CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Mar 22 14:16:50 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
Resurrect the counter stuff from HEAD, which tries to deal with 32-bit
counter rollover. Drive it by sysctl and/or callout every 5 minutes,
since there's no garbage collection kthread any more.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.16 -r1.126.2.17 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.16 src/sys/kern/vfs_cache.c:1.126.2.17
--- src/sys/kern/vfs_cache.c:1.126.2.16 Sun Mar 22 01:58:22 2020
+++ src/sys/kern/vfs_cache.c Sun Mar 22 14:16:50 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.16 2020/03/22 01:58:22 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.17 2020/03/22 14:16:50 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -153,9 +153,10 @@
* and the definition of "struct vnode" for the particulars.
*
* Per-CPU statistics, and LRU list totals are read unlocked, since
- * an approximate value is OK. We maintain uintptr_t sized per-CPU
- * counters and 64-bit global counters under the theory that uintptr_t
- * sized counters are less likely to be hosed by nonatomic increment.
+ * an approximate value is OK. We maintain 32-bit sized per-CPU
+ * counters and 64-bit global counters under the theory that 32-bit
+ * sized counters are less likely to be hosed by nonatomic increment
+ * (on 32-bit platforms).
*
* The lock order is:
*
@@ -171,7 +172,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.16 2020/03/22 01:58:22 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.17 2020/03/22 14:16:50 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -180,6 +181,8 @@ __KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,
#endif
#include
+#include
+#include
#include
#include
#include
@@ -199,15 +202,13 @@ __KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,
#include
static void cache_activate(struct namecache *);
+static void cache_update_stats(void *);
static int cache_compare_key(void *, const void *, const void *);
static int cache_compare_nodes(void *, const void *, const void *);
static void cache_deactivate(void);
static void cache_reclaim(void);
static int cache_stat_sysctl(SYSCTLFN_ARGS);
-/* Per-CPU counters. */
-struct nchstats_percpu _NAMEI_CACHE_STATS(uintptr_t);
-
/* Global pool cache. */
static pool_cache_t cache_pool __read_mostly;
@@ -225,8 +226,15 @@ static struct {
static kmutex_t cache_lru_lock __cacheline_aligned;
-/* Cache effectiveness statistics. This holds total from per-cpu stats */
-struct nchstats nchstats __cacheline_aligned;
+/* Cache effectiveness statistics. nchstats holds system-wide total. */
+struct nchstats nchstats;
+struct nchstats_percpu _NAMEI_CACHE_STATS(uint32_t);
+struct nchcpu {
+ struct nchstats_percpu cur;
+ struct nchstats_percpu last;
+};
+static callout_t cache_stat_callout;
+static kmutex_t cache_stat_lock __cacheline_aligned;
#define COUNT(f) do { \
lwp_t *l = curlwp; \
@@ -235,6 +243,12 @@ struct nchstats nchstats __cacheline_ali
KPREEMPT_ENABLE(l); \
} while (/* CONSTCOND */ 0);
+#define UPDATE(nchcpu, f) do { \
+ uint32_t cur = atomic_load_relaxed(&nchcpu->cur.f); \
+ nchstats.f += cur - nchcpu->last.f; \
+ nchcpu->last.f = cur; \
+} while (/* CONSTCOND */ 0)
+
/*
* Tunables. cache_maxlen replaces the historical doingcache:
* set it zero to disable caching for debugging purposes.
@@ -242,6 +256,7 @@ struct nchstats nchstats __cacheline_ali
int cache_lru_maxdeact __read_mostly = 2; /* max # to deactivate */
int cache_lru_maxscan __read_mostly = 64; /* max # to scan/reclaim */
int cache_maxlen __read_mostly = USHRT_MAX; /* max name length to cache */
+int cache_stat_interval __read_mostly = 300; /* in seconds */
/* sysctl */
static struct sysctllog *cache_sysctllog;
@@ -1000,6 +1015,11 @@ nchinit(void)
TAILQ_INIT(&cache_lru.list[LRU_ACTIVE]);
TAILQ_INIT(&cache_lru.list[LRU_INACTIVE]);
+ mutex_init(&cache_stat_lock, MUTEX_DEFAULT, IPL_NONE);
+ callout_init(&cache_stat_callout, CALLOUT_MPSAFE);
+ callout_setfunc(&cache_stat_callout, cache_update_stats, NULL);
+ callout_schedule(&cache_stat_callout, cache_stat_interval * hz);
+
KASSERT(cache_sysctllog == NULL);
sysctl_createv(&cache_sysctllog, 0, NULL, NULL,
CTLFLAG_PERMANENT,
@@ -1362,6 +1382,41 @@ namecache_count_2passes(void)
}
/*
+ * Sum the stats from all CPUs into nchstats. This needs to run at least
+ * once within every window where a 32-bit counter could roll over. It's
+ * called regularly by timer to ensure this.
+ */
+static void
+cache_update_stats(void *cookie)
+{
+ CPU_INFO_ITERATOR cii;
+ struct cpu_info *ci;
+
+ mutex_enter(&cache_stat_lock);
+ for (CPU_INFO_FOREACH(cii, ci)) {
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Mar 22 01:58:22 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
- rb_tree_insert_node() refuses to insert duplicate nodes. It always
returns a pointer to the current node in the tree for the given key.
Therefore the call to rb_tree_lookup_node() in cache_enter() is not
needed.
- Adjust a comment and move some code for clarity.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.15 -r1.126.2.16 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.15 src/sys/kern/vfs_cache.c:1.126.2.16
--- src/sys/kern/vfs_cache.c:1.126.2.15 Sun Mar 22 00:34:31 2020
+++ src/sys/kern/vfs_cache.c Sun Mar 22 01:58:22 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.15 2020/03/22 00:34:31 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.16 2020/03/22 01:58:22 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.15 2020/03/22 00:34:31 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.16 2020/03/22 01:58:22 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -416,8 +416,6 @@ cache_lookup_entry(struct vnode *dvp, co
* Search the RB tree for the key. This is an inlined lookup
* tailored for exactly what's needed here (64-bit key and so on)
* that is quite a bit faster than using rb_tree_find_node().
- * Elsewhere during entry/removal the usual functions are used as it
- * doesn't matter there.
*/
for (;;) {
if (__predict_false(RB_SENTINEL_P(node))) {
@@ -525,6 +523,9 @@ cache_lookup(struct vnode *dvp, const ch
return false;
}
+ /* Compute the key up front - don't need the lock. */
+ key = cache_key(name, namelen);
+
/* Could the entry be purged below? */
if ((cnflags & ISLASTCN) != 0 &&
((cnflags & MAKEENTRY) == 0 || nameiop == CREATE)) {
@@ -533,9 +534,6 @@ cache_lookup(struct vnode *dvp, const ch
op = RW_READER;
}
- /* Compute the key up front - don't need the lock. */
- key = cache_key(name, namelen);
-
/* Now look for the name. */
rw_enter(&dvi->vi_nc_lock, op);
ncp = cache_lookup_entry(dvp, name, namelen, key);
@@ -890,18 +888,20 @@ cache_enter(struct vnode *dvp, struct vn
/*
* Insert to the directory. Concurrent lookups in the same
* directory may race for a cache entry. There can also be hash
- * value collisions. If there's a entry there already, free it.
+ * value collisions. If there's a entry there already, purge it.
*/
rw_enter(&dvi->vi_nc_lock, RW_WRITER);
- oncp = rb_tree_find_node(&dvi->vi_nc_tree, &ncp->nc_key);
- if (oncp) {
+ oncp = rb_tree_insert_node(&dvi->vi_nc_tree, ncp);
+ if (oncp != ncp) {
+ KASSERT(oncp->nc_key == ncp->nc_key);
KASSERT(oncp->nc_nlen == ncp->nc_nlen);
if (cache_namecmp(oncp, name, namelen)) {
COUNT(ncs_collisions);
}
cache_remove(oncp, true);
+ oncp = rb_tree_insert_node(&dvi->vi_nc_tree, ncp);
+ KASSERT(oncp == ncp);
}
- rb_tree_insert_node(&dvi->vi_nc_tree, ncp);
/* Then insert to the vnode. */
if (vp == NULL) {
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Mar 22 00:34:31 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
- Fold the "doingcache" conditional into the max length conditional.
- Mark the tunables __read_mostly.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.14 -r1.126.2.15 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.14 src/sys/kern/vfs_cache.c:1.126.2.15
--- src/sys/kern/vfs_cache.c:1.126.2.14 Sat Mar 21 22:00:55 2020
+++ src/sys/kern/vfs_cache.c Sun Mar 22 00:34:31 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.14 2020/03/21 22:00:55 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.15 2020/03/22 00:34:31 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.14 2020/03/21 22:00:55 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.15 2020/03/22 00:34:31 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -235,10 +235,13 @@ struct nchstats nchstats __cacheline_ali
KPREEMPT_ENABLE(l); \
} while (/* CONSTCOND */ 0);
-/* Tunables */
-static const int cache_lru_maxdeact = 2; /* max # to deactivate */
-static const int cache_lru_maxscan = 64; /* max # to scan/reclaim */
-static int doingcache = 1; /* 1 => enable the cache */
+/*
+ * Tunables. cache_maxlen replaces the historical doingcache:
+ * set it zero to disable caching for debugging purposes.
+ */
+int cache_lru_maxdeact __read_mostly = 2; /* max # to deactivate */
+int cache_lru_maxscan __read_mostly = 64; /* max # to scan/reclaim */
+int cache_maxlen __read_mostly = USHRT_MAX; /* max name length to cache */
/* sysctl */
static struct sysctllog *cache_sysctllog;
@@ -515,11 +518,7 @@ cache_lookup(struct vnode *dvp, const ch
}
*vn_ret = NULL;
- if (__predict_false(!doingcache)) {
- return false;
- }
-
- if (__predict_false(namelen > USHRT_MAX)) {
+ if (__predict_false(namelen > cache_maxlen)) {
SDT_PROBE(vfs, namecache, lookup, toolong, dvp,
name, namelen, 0, 0);
COUNT(ncs_long);
@@ -644,7 +643,7 @@ cache_lookup_linked(struct vnode *dvp, c
*vn_ret = NULL;
/* If disabled, or file system doesn't support this, bail out. */
- if (__predict_false(!doingcache ||
+ if (__predict_false(cache_maxlen == 0 ||
(dvp->v_mount->mnt_iflag & IMNT_NCLOOKUP) == 0)) {
return false;
}
@@ -753,7 +752,7 @@ cache_revlookup(struct vnode *vp, struct
KASSERT(vp != NULL);
- if (!doingcache)
+ if (cache_maxlen == 0)
goto out;
rw_enter(&vi->vi_nc_listlock, RW_READER);
@@ -855,7 +854,7 @@ cache_enter(struct vnode *dvp, struct vn
/* First, check whether we can/should add a cache entry. */
if ((cnflags & MAKEENTRY) == 0 ||
- __predict_false(namelen > USHRT_MAX || !doingcache)) {
+ __predict_false(namelen > cache_maxlen)) {
SDT_PROBE(vfs, namecache, enter, toolong, vp, name, namelen,
0, 0);
return;
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sat Mar 21 22:00:56 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
- Include sys/types.h for vax.
- Use macro versions of kpreempt_disable()/enable().
- Use atomic_load_relaxed().
- Make cache_key() inline.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.13 -r1.126.2.14 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.13 src/sys/kern/vfs_cache.c:1.126.2.14
--- src/sys/kern/vfs_cache.c:1.126.2.13 Tue Mar 10 21:53:45 2020
+++ src/sys/kern/vfs_cache.c Sat Mar 21 22:00:55 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.13 2020/03/10 21:53:45 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.14 2020/03/21 22:00:55 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.13 2020/03/10 21:53:45 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.14 2020/03/21 22:00:55 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -179,6 +179,7 @@ __KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,
#include "opt_dtrace.h"
#endif
+#include
#include
#include
#include
@@ -228,9 +229,10 @@ static kmutex_t cache_lru_lock __cacheli
struct nchstats nchstats __cacheline_aligned;
#define COUNT(f) do { \
- kpreempt_disable(); \
+ lwp_t *l = curlwp; \
+ KPREEMPT_DISABLE(l); \
((struct nchstats_percpu *)curcpu()->ci_data.cpu_nch)->f++; \
- kpreempt_enable(); \
+ KPREEMPT_ENABLE(l); \
} while (/* CONSTCOND */ 0);
/* Tunables */
@@ -314,7 +316,7 @@ cache_compare_key(void *context, const v
* the key value to try and improve uniqueness, and so that length doesn't
* need to be compared separately for string comparisons.
*/
-static int64_t
+static inline int64_t
cache_key(const char *name, size_t nlen)
{
int64_t key;
@@ -403,6 +405,7 @@ cache_lookup_entry(struct vnode *dvp, co
vnode_impl_t *dvi = VNODE_TO_VIMPL(dvp);
struct rb_node *node = dvi->vi_nc_tree.rbt_root;
struct namecache *ncp;
+ int lrulist;
KASSERT(rw_lock_held(&dvi->vi_nc_lock));
@@ -436,7 +439,8 @@ cache_lookup_entry(struct vnode *dvp, co
* unlocked check, but it will rarely be wrong and even then there
* will be no harm caused.
*/
- if (__predict_false(ncp->nc_lrulist != LRU_ACTIVE)) {
+ lrulist = atomic_load_relaxed(&ncp->nc_lrulist);
+ if (__predict_false(lrulist != LRU_ACTIVE)) {
cache_activate(ncp);
}
return ncp;
@@ -744,8 +748,8 @@ cache_revlookup(struct vnode *vp, struct
vnode_impl_t *vi = VNODE_TO_VIMPL(vp);
struct namecache *ncp;
struct vnode *dvp;
+ int error, nlen, lrulist;
char *bp;
- int error, nlen;
KASSERT(vp != NULL);
@@ -793,7 +797,8 @@ cache_revlookup(struct vnode *vp, struct
}
/* Record a hit on the entry. This is an unlocked read. */
- if (ncp->nc_lrulist != LRU_ACTIVE) {
+ lrulist = atomic_load_relaxed(&ncp->nc_lrulist);
+ if (lrulist != LRU_ACTIVE) {
cache_activate(ncp);
}
@@ -863,7 +868,8 @@ cache_enter(struct vnode *dvp, struct vn
* but it doesn't matter. Just need to catch up with things
* eventually: it doesn't matter if we go over temporarily.
*/
- total = cache_lru.count[LRU_ACTIVE] + cache_lru.count[LRU_INACTIVE];
+ total = atomic_load_relaxed(&cache_lru.count[LRU_ACTIVE]);
+ total += atomic_load_relaxed(&cache_lru.count[LRU_INACTIVE]);
if (__predict_false(total > desiredvnodes)) {
cache_reclaim();
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Tue Mar 10 21:53:46 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
__read_mostly -> const in one place
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.12 -r1.126.2.13 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.12 src/sys/kern/vfs_cache.c:1.126.2.13
--- src/sys/kern/vfs_cache.c:1.126.2.12 Sun Feb 16 22:00:53 2020
+++ src/sys/kern/vfs_cache.c Tue Mar 10 21:53:45 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.12 2020/02/16 22:00:53 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.13 2020/03/10 21:53:45 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -171,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.12 2020/02/16 22:00:53 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.13 2020/03/10 21:53:45 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -242,7 +242,7 @@ static int doingcache = 1; /* 1 => ena
static struct sysctllog *cache_sysctllog;
/* Read-black tree */
-static rb_tree_ops_t cache_rbtree_ops __read_mostly = {
+static const rb_tree_ops_t cache_rbtree_ops = {
.rbto_compare_nodes = cache_compare_nodes,
.rbto_compare_key = cache_compare_key,
.rbto_node_offset = offsetof(struct namecache, nc_tree),
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Tue Mar 3 22:30:57 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_lookup.c
Log Message:
lookup_fastforward(): bail out on mount -o union
To generate a diff of this commit:
cvs rdiff -u -r1.212.4.10 -r1.212.4.11 src/sys/kern/vfs_lookup.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.212.4.10 src/sys/kern/vfs_lookup.c:1.212.4.11
--- src/sys/kern/vfs_lookup.c:1.212.4.10 Sat Feb 29 20:21:03 2020
+++ src/sys/kern/vfs_lookup.c Tue Mar 3 22:30:57 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.212.4.10 2020/02/29 20:21:03 ad Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.212.4.11 2020/03/03 22:30:57 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.10 2020/02/29 20:21:03 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.11 2020/03/03 22:30:57 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -1308,6 +1308,13 @@ lookup_fastforward(struct namei_state *s
}
}
+ /* Can't deal with -o union lookups. */
+ if ((searchdir->v_vflag & VV_ROOT) != 0 &&
+ (searchdir->v_mount->mnt_flag & MNT_UNION) != 0) {
+ error = EOPNOTSUPP;
+ break;
+ }
+
/*
* Good, now look for it in cache. cache_lookup_linked()
* will fail if there's nothing there, or if there's no
@@ -1323,7 +1330,6 @@ lookup_fastforward(struct namei_state *s
/* Scored a hit. Negative is good too (ENOENT). */
if (foundobj == NULL) {
- /* XXXAD need to handle -o union mount. */
error = ENOENT;
break;
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sat Feb 29 22:00:03 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_vnops.c
Log Message:
Back out experimental change - not ready for LK_SHARED on VOP_OPEN() just yet.
To generate a diff of this commit:
cvs rdiff -u -r1.204.2.3 -r1.204.2.4 src/sys/kern/vfs_vnops.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_vnops.c
diff -u src/sys/kern/vfs_vnops.c:1.204.2.3 src/sys/kern/vfs_vnops.c:1.204.2.4
--- src/sys/kern/vfs_vnops.c:1.204.2.3 Sat Feb 29 20:21:03 2020
+++ src/sys/kern/vfs_vnops.c Sat Feb 29 22:00:03 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnops.c,v 1.204.2.3 2020/02/29 20:21:03 ad Exp $ */
+/* $NetBSD: vfs_vnops.c,v 1.204.2.4 2020/02/29 22:00:03 ad Exp $ */
/*-
* Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -66,7 +66,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.204.2.3 2020/02/29 20:21:03 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.204.2.4 2020/02/29 22:00:03 ad Exp $");
#include "veriexec.h"
@@ -163,10 +163,7 @@ vn_open(struct nameidata *ndp, int fmode
ndp->ni_cnd.cn_flags |= FOLLOW;
} else {
ndp->ni_cnd.cn_nameiop = LOOKUP;
- if ((fmode & O_TRUNC) == 0) /* XXXAD check nfs etc */
- ndp->ni_cnd.cn_flags |= LOCKLEAF | LOCKSHARED;
- else
- ndp->ni_cnd.cn_flags |= LOCKLEAF;
+ ndp->ni_cnd.cn_flags |= LOCKLEAF;
if ((fmode & O_NOFOLLOW) == 0)
ndp->ni_cnd.cn_flags |= FOLLOW;
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Feb 23 19:14:03 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_vnode.c
Log Message:
Turns out there's no point adjusting v_holdcnt with atomics.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.7 -r1.105.2.8 src/sys/kern/vfs_vnode.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_vnode.c
diff -u src/sys/kern/vfs_vnode.c:1.105.2.7 src/sys/kern/vfs_vnode.c:1.105.2.8
--- src/sys/kern/vfs_vnode.c:1.105.2.7 Sat Jan 25 22:38:51 2020
+++ src/sys/kern/vfs_vnode.c Sun Feb 23 19:14:03 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnode.c,v 1.105.2.7 2020/01/25 22:38:51 ad Exp $ */
+/* $NetBSD: vfs_vnode.c,v 1.105.2.8 2020/02/23 19:14:03 ad Exp $ */
/*-
* Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
@@ -142,10 +142,10 @@
* as vput(9), routines. Common points holding references are e.g.
* file openings, current working directory, mount points, etc.
*
- * Note on v_usecount & v_holdcnt and their locking
+ * Note on v_usecount and its locking
*
- * At nearly all points it is known that the counts could be zero,
- * the vnode_t::v_interlock will be held. To change the counts away
+ * At nearly all points it is known that v_usecount could be zero,
+ * the vnode_t::v_interlock will be held. To change the count away
* from zero, the interlock must be held. To change from a non-zero
* value to zero, again the interlock must be held.
*
@@ -154,7 +154,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.7 2020/01/25 22:38:51 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.8 2020/02/23 19:14:03 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_pax.h"
@@ -939,7 +939,7 @@ vholdl(vnode_t *vp)
KASSERT(mutex_owned(vp->v_interlock));
- if (atomic_inc_uint_nv(&vp->v_holdcnt) == 1 && vp->v_usecount == 0)
+ if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
lru_requeue(vp, lru_which(vp));
}
@@ -949,17 +949,6 @@ vholdl(vnode_t *vp)
void
vhold(vnode_t *vp)
{
- int hold, next;
-
- for (hold = vp->v_holdcnt;; hold = next) {
- if (__predict_false(hold == 0)) {
- break;
- }
- next = atomic_cas_uint(&vp->v_holdcnt, hold, hold + 1);
- if (__predict_true(next == hold)) {
- return;
- }
- }
mutex_enter(vp->v_interlock);
vholdl(vp);
@@ -980,7 +969,8 @@ holdrelel(vnode_t *vp)
vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
}
- if (atomic_dec_uint_nv(&vp->v_holdcnt) == 0 && vp->v_usecount == 0)
+ vp->v_holdcnt--;
+ if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
lru_requeue(vp, lru_which(vp));
}
@@ -990,18 +980,6 @@ holdrelel(vnode_t *vp)
void
holdrele(vnode_t *vp)
{
- int hold, next;
-
- for (hold = vp->v_holdcnt;; hold = next) {
- if (__predict_false(hold == 1)) {
- break;
- }
- KASSERT(hold > 1);
- next = atomic_cas_uint(&vp->v_holdcnt, hold, hold - 1);
- if (__predict_true(next == hold)) {
- return;
- }
- }
mutex_enter(vp->v_interlock);
holdrelel(vp);
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Feb 16 22:00:53 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
Comments.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.11 -r1.126.2.12 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.11 src/sys/kern/vfs_cache.c:1.126.2.12
--- src/sys/kern/vfs_cache.c:1.126.2.11 Fri Jan 24 16:48:59 2020
+++ src/sys/kern/vfs_cache.c Sun Feb 16 22:00:53 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.11 2020/01/24 16:48:59 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.12 2020/02/16 22:00:53 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -67,8 +67,9 @@
* reference. It is managed LRU, so frequently used names will hang
* around. The cache is indexed by hash value obtained from the name.
*
- * The name cache (or directory name lookup cache) is the brainchild of
- * Robert Elz and made its first appearance in 4.3BSD.
+ * The name cache is the brainchild of Robert Elz and was introduced in
+ * 4.3BSD. See "Using gprof to Tune the 4.2BSD Kernel", Marshall Kirk
+ * McKusick, May 21 1984.
*
* Data structures:
*
@@ -81,7 +82,7 @@
* The index is a red-black tree. There are no special concurrency
* requirements placed on it, because it's per-directory and protected
* by the namecache's per-directory locks. It should therefore not be
- * difficult to experiment with other data structures.
+ * difficult to experiment with other types of index.
*
* Each cached name is stored in a struct namecache, along with a
* pointer to the associated vnode (nc_vp). Names longer than a
@@ -170,7 +171,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.11 2020/01/24 16:48:59 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.12 2020/02/16 22:00:53 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -393,7 +394,7 @@ cache_remove(struct namecache *ncp, cons
}
/*
- * Find a single cache entry and return it. The vnode lock must be held.
+ * Find a single cache entry and return it. vi_nc_lock must be held.
*/
static struct namecache * __noinline
cache_lookup_entry(struct vnode *dvp, const char *name, size_t namelen,
@@ -1239,11 +1240,10 @@ cache_deactivate(void)
}
/*
- * Aim for a 1:1 ratio of active to inactive. It's a bit wet finger
- * in the air here, but this is to allow each potential victim a
- * reasonable amount of time to cycle through the inactive list in
- * order to score a hit and be reactivated, while trying not to
- * cause reactivations too frequently.
+ * Aim for a 1:1 ratio of active to inactive. This is to allow each
+ * potential victim a reasonable amount of time to cycle through the
+ * inactive list in order to score a hit and be reactivated, while
+ * trying not to cause reactivations too frequently.
*/
if (cache_lru.count[LRU_ACTIVE] < cache_lru.count[LRU_INACTIVE]) {
return;
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sat Jan 25 18:42:24 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cwd.c
Log Message:
Add a missing membar to previous.
To generate a diff of this commit:
cvs rdiff -u -r1.4.62.1 -r1.4.62.2 src/sys/kern/vfs_cwd.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cwd.c
diff -u src/sys/kern/vfs_cwd.c:1.4.62.1 src/sys/kern/vfs_cwd.c:1.4.62.2
--- src/sys/kern/vfs_cwd.c:1.4.62.1 Sat Jan 25 15:54:03 2020
+++ src/sys/kern/vfs_cwd.c Sat Jan 25 18:42:24 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cwd.c,v 1.4.62.1 2020/01/25 15:54:03 ad Exp $ */
+/* $NetBSD: vfs_cwd.c,v 1.4.62.2 2020/01/25 18:42:24 ad Exp $ */
/*-
* Copyright (c) 2008, 2020 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cwd.c,v 1.4.62.1 2020/01/25 15:54:03 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cwd.c,v 1.4.62.2 2020/01/25 18:42:24 ad Exp $");
#include
#include
@@ -183,8 +183,10 @@ cwdenter(krw_t op)
* changes while the caller is pondering the cwdinfo.
*/
kpreempt_disable();
- if (__predict_true(mutex_owner(&cwdi->cwdi_lock) == NULL))
+ if (__predict_true(mutex_owner(&cwdi->cwdi_lock) == NULL)) {
+ membar_consumer();
return cwdi;
+ }
kpreempt_enable();
mutex_enter(&cwdi->cwdi_lock);
} else {
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Fri Jan 24 16:05:37 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_lookup.c
Log Message:
Add a comment.
To generate a diff of this commit:
cvs rdiff -u -r1.212.4.7 -r1.212.4.8 src/sys/kern/vfs_lookup.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.212.4.7 src/sys/kern/vfs_lookup.c:1.212.4.8
--- src/sys/kern/vfs_lookup.c:1.212.4.7 Thu Jan 23 21:23:56 2020
+++ src/sys/kern/vfs_lookup.c Fri Jan 24 16:05:37 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.212.4.7 2020/01/23 21:23:56 ad Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.212.4.8 2020/01/24 16:05:37 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.7 2020/01/23 21:23:56 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.8 2020/01/24 16:05:37 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -1324,6 +1324,7 @@ lookup_fastforward(struct namei_state *s
/* Scored a hit. Negative is good too (ENOENT). */
if (foundobj == NULL) {
+ /* XXXAD need to handle -o union mount. */
error = ENOENT;
break;
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src Committed By: ad Date: Thu Jan 23 21:23:56 UTC 2020 Modified Files: src/sys/kern [ad-namecache]: vfs_lookup.c Log Message: Improve an assertion. To generate a diff of this commit: cvs rdiff -u -r1.212.4.6 -r1.212.4.7 src/sys/kern/vfs_lookup.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: src/sys/kern/vfs_lookup.c diff -u src/sys/kern/vfs_lookup.c:1.212.4.6 src/sys/kern/vfs_lookup.c:1.212.4.7 --- src/sys/kern/vfs_lookup.c:1.212.4.6 Thu Jan 23 12:21:01 2020 +++ src/sys/kern/vfs_lookup.c Thu Jan 23 21:23:56 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: vfs_lookup.c,v 1.212.4.6 2020/01/23 12:21:01 ad Exp $ */ +/* $NetBSD: vfs_lookup.c,v 1.212.4.7 2020/01/23 21:23:56 ad Exp $ */ /* * Copyright (c) 1982, 1986, 1989, 1993 @@ -37,7 +37,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.6 2020/01/23 12:21:01 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.7 2020/01/23 21:23:56 ad Exp $"); #ifdef _KERNEL_OPT #include "opt_magiclinks.h" @@ -1390,7 +1390,7 @@ lookup_fastforward(struct namei_state *s rw_exit(plock); } - KASSERT(foundobj == NULL || error == 0); + KASSERT(error == 0 ? foundobj != NULL : foundobj == NULL); *foundobj_ret = foundobj; return error; }
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Thu Jan 23 19:28:39 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_vnode.c
Log Message:
vcache_reclaim(): purge namecache immediately after setting vnode to
VS_RECLAIMED.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.3 -r1.105.2.4 src/sys/kern/vfs_vnode.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_vnode.c
diff -u src/sys/kern/vfs_vnode.c:1.105.2.3 src/sys/kern/vfs_vnode.c:1.105.2.4
--- src/sys/kern/vfs_vnode.c:1.105.2.3 Fri Jan 17 21:55:13 2020
+++ src/sys/kern/vfs_vnode.c Thu Jan 23 19:28:39 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $ */
+/* $NetBSD: vfs_vnode.c,v 1.105.2.4 2020/01/23 19:28:39 ad Exp $ */
/*-
* Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
@@ -145,7 +145,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.4 2020/01/23 19:28:39 ad Exp $");
#include
#include
@@ -1582,6 +1582,13 @@ vcache_reclaim(vnode_t *vp)
vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
mutex_exit(vp->v_interlock);
+ /*
+ * With vnode state set to reclaiming, purge name cache immediately
+ * to prevent new handles on vnode, and wait for existing threads
+ * trying to get a handle to notice VS_RECLAIMED status and abort.
+ */
+ cache_purge(vp);
+
/* Replace the vnode key with a temporary copy. */
if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
@@ -1635,9 +1642,6 @@ vcache_reclaim(vnode_t *vp)
vp->v_ractx = NULL;
}
- /* Purge name cache. */
- cache_purge(vp);
-
if (vip->vi_key.vk_key_len > 0) {
/* Remove from vnode cache. */
hash = vcache_hash(&vip->vi_key);
CVS commit: [ad-namecache] src/sys/kern
Module Name:src Committed By: ad Date: Thu Jan 23 12:33:18 UTC 2020 Modified Files: src/sys/kern [ad-namecache]: vfs_cache.c Log Message: Update comments. To generate a diff of this commit: cvs rdiff -u -r1.126.2.9 -r1.126.2.10 src/sys/kern/vfs_cache.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: src/sys/kern/vfs_cache.c diff -u src/sys/kern/vfs_cache.c:1.126.2.9 src/sys/kern/vfs_cache.c:1.126.2.10 --- src/sys/kern/vfs_cache.c:1.126.2.9 Sun Jan 19 21:19:25 2020 +++ src/sys/kern/vfs_cache.c Thu Jan 23 12:33:18 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: vfs_cache.c,v 1.126.2.9 2020/01/19 21:19:25 ad Exp $ */ +/* $NetBSD: vfs_cache.c,v 1.126.2.10 2020/01/23 12:33:18 ad Exp $ */ /*- * Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc. @@ -63,92 +63,104 @@ /* * Name caching: * - * Names found by directory scans are retained in a cache for future - * reference. It is managed pseudo-LRU, so frequently used names will - * hang around. Cache is indexed by directory. - * - * Upon reaching the last segment of a path, if the reference is for - * DELETE, or NOCACHE is set (rewrite), and name is located in the - * cache, it will be dropped. + * Names found by directory scans are retained in a cache for future + * reference. It is managed pseudo-LRU, so frequently used names will + * hang around. The cache is indexed by directory. * * Background: * * XXX add a bit of history * - * Data structures: + * Data structures * - * The original BSD implementation used a global hash table, which - * works very well but can cause difficulties for the CPU cache on - * modern CPUs, and concurrency headaches for the kernel hacker on - * multiprocessor systems. The global hash table is also difficult to - * size dynamically. To try and address these concerns, the focus of - * interest in this implementation is the directory itself: a - * per-directory red-black tree is used to look up names. Other than - * the choice of data structure, it works largely the same way as the - * BSD implementation. + * Typically DNLCs use a global hash table for lookup, which works very + * well but can cause challenges for the CPU cache on modern CPUs, and + * concurrency headaches for the kernel hacker. The focus of interest + * in this implementation is the directory itself: a per-directory data + * structure is used to index names. Currently this is a red-black + * tree. There are no special concurrency requirements placed on the + * index, so it should not be difficult to experiment with other + * structures. + * + * Each cached name is stored in a struct namecache, along with a + * pointer the associated vnode (nc_vp). For simplicity (and economy + * of storage), names longer than a maximum length of NCHNAMLEN are + * allocated with kmem_alloc(); they occur infrequently. Names shorter + * than this are stored directly in struct namecache. If it is a + * "negative" entry, (i.e. for a name that is known NOT to exist) the + * vnode pointer will be NULL. * - * Replacement: - * - * XXX LRU blurb. - * - * Concurrency: - * - * XXX need new blurb here - * - * See definition of "struct namecache" in src/sys/namei.src for the - * particulars. - * - * Per-CPU statistics, and "numcache" are read unlocked, since an - * approximate value is OK. We maintain uintptr_t sized per-CPU - * counters and 64-bit global counters under the theory that uintptr_t - * sized counters are less likely to be hosed by nonatomic increment. - * - * Lock order: - * - * 1) nc_dvp->vi_ncdlock - * 2) nc_dvp->vi_ncvlock - * 3) cache_lru_lock, vp->v_interlock - * - * Ugly ASCII diagram: - * - * XXX replace tabs with spaces, make less ugly + * The nchnode (XXXAD vnode) and namecache structures are connected together thusly + * (the root is at the bottom of the diagram): * * ... * ^ - * | - * -o- - * | VDIR | - * | nchnode | - * --- - * ^ - * |- nd_tree + * |- nn_tree * | - * -o- --- --- + * +o+ +-+ +-+ * | VDIR | | VCHR | | VREG | * | nchnode o-+ | nchnode o-+ | nchnode o--+ - * --- | --- | --- | + * +-+ | +-+ | +-+ | * ^ | ^ | ^ | * |- nc_nn |- nn_list |- nc_nn |- nn_list |- nc_nn| * | | | | | | - * -o- | -o- | -o- | + * +o+ |
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Thu Jan 23 12:21:01 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_lookup.c
Log Message:
- Change style of new code slightly to match rest of file.
- NFS lookup needs to cross mountpoint too.
- Update comments.
To generate a diff of this commit:
cvs rdiff -u -r1.212.4.5 -r1.212.4.6 src/sys/kern/vfs_lookup.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.212.4.5 src/sys/kern/vfs_lookup.c:1.212.4.6
--- src/sys/kern/vfs_lookup.c:1.212.4.5 Wed Jan 22 12:10:46 2020
+++ src/sys/kern/vfs_lookup.c Thu Jan 23 12:21:01 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.212.4.5 2020/01/22 12:10:46 ad Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.212.4.6 2020/01/23 12:21:01 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.5 2020/01/22 12:10:46 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.6 2020/01/23 12:21:01 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -1113,9 +1113,9 @@ lookup_once(struct namei_state *state,
/*
* If the file system supports VOP_LOOKUP() with a shared lock, and
* we are not making any modifications (nameiop LOOKUP) or this is
- * not the last component then get a shared lock LK_SHARED. Where
- * we can't do fast-forwarded lookups (for example with layered file
- * systems) then this is the fallback for reducing lock contention.
+ * not the last component then get a shared lock. Where we can't do
+ * fast-forwarded lookups (for example with layered file systems)
+ * then this is the fallback for reducing lock contention.
*/
if ((searchdir->v_mount->mnt_iflag & IMNT_SHRLOOKUP) != 0 &&
(cnp->cn_nameiop == LOOKUP || (cnp->cn_flags & ISLASTCN) == 0)) {
@@ -1251,13 +1251,13 @@ done:
* final vnode will have its reference count adjusted and lock taken.
*/
static int
-lookup_fastforward(struct namei_state *state, struct vnode **searchdir,
- struct vnode **foundobj)
+lookup_fastforward(struct namei_state *state, struct vnode **searchdir_ret,
+ struct vnode **foundobj_ret)
{
struct componentname *cnp = state->cnp;
struct nameidata *ndp = state->ndp;
krwlock_t *plock;
- struct vnode *vp, *origsearchdir;
+ struct vnode *foundobj, *searchdir;
int error, error2;
size_t oldpathlen;
const char *oldnameptr;
@@ -1268,10 +1268,12 @@ lookup_fastforward(struct namei_state *s
* case we can't get vnode references and need to roll back.
*/
plock = NULL;
- origsearchdir = *searchdir;
+ searchdir = *searchdir_ret;
oldnameptr = cnp->cn_nameptr;
oldpathlen = ndp->ni_pathlen;
for (;;) {
+ foundobj = NULL;
+
/*
* Get the next component name. There should be no slashes
* here, and we shouldn't have looped around if we were
@@ -1289,15 +1291,15 @@ lookup_fastforward(struct namei_state *s
* that need to be made. Also check for missing mountpoints.
*/
if ((cnp->cn_flags & ISDOTDOT) != 0 ||
- (*searchdir)->v_mount == NULL) {
+ searchdir->v_mount == NULL) {
error = EOPNOTSUPP;
break;
}
/*
* Can't deal with last component when modifying; this needs
- * the directory vnode locked and VOP_LOOKUP() called (which
- * can and does modify state, despite the name).
+ * searchdir locked and VOP_LOOKUP() called (which can and
+ * does modify state, despite the name).
*/
if ((cnp->cn_flags & ISLASTCN) != 0) {
if (cnp->cn_nameiop != LOOKUP ||
@@ -1313,16 +1315,15 @@ lookup_fastforward(struct namei_state *s
* ownership info for the directory, or if the user doesn't
* have permission to look up files in this directory.
*/
- if (!cache_lookup_linked(*searchdir, cnp->cn_nameptr,
- cnp->cn_namelen, &vp, &plock, cnp->cn_cred)) {
+ if (!cache_lookup_linked(searchdir, cnp->cn_nameptr,
+ cnp->cn_namelen, &foundobj, &plock, cnp->cn_cred)) {
error = EOPNOTSUPP;
break;
}
KASSERT(plock != NULL && rw_lock_held(plock));
/* Scored a hit. Negative is good too (ENOENT). */
- if (vp == NULL) {
- *foundobj = vp;
+ if (foundobj == NULL) {
error = ENOENT;
break;
}
@@ -1335,15 +1336,16 @@ lookup_fastforward(struct namei_state *s
* - or encountered a mount point that needs to be crossed.
* - or encountered something other than a directory.
*/
- if ((cnp->cn_flags & ISLASTCN) != 0 || vp->v_type != VDIR ||
- (vp->v_type == VDIR && vp->v_mountedhere != NULL)) {
- mutex_enter(vp->v_interlock);
- error = vcache_tryvget(vp);
+ if ((cnp->cn_flags & ISLASTCN) != 0 ||
+ foundobj->v_type != VDIR ||
+ (foundobj->v_type == VDIR &&
+ foundobj->v_mountedhere != NULL)) {
+ mutex_enter(foundobj->v_interlock);
+ error = vcache_tryvget(foundobj);
/* v_interlock now released */
- if (error ==
CVS commit: [ad-namecache] src/sys/kern
Module Name:src Committed By: ad Date: Thu Jan 23 12:17:08 UTC 2020 Modified Files: src/sys/kern [ad-namecache]: kern_synch.c Log Message: Back out previous. To generate a diff of this commit: cvs rdiff -u -r1.334.2.2 -r1.334.2.3 src/sys/kern/kern_synch.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: src/sys/kern/kern_synch.c diff -u src/sys/kern/kern_synch.c:1.334.2.2 src/sys/kern/kern_synch.c:1.334.2.3 --- src/sys/kern/kern_synch.c:1.334.2.2 Sun Jan 19 21:08:29 2020 +++ src/sys/kern/kern_synch.c Thu Jan 23 12:17:08 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: kern_synch.c,v 1.334.2.2 2020/01/19 21:08:29 ad Exp $ */ +/* $NetBSD: kern_synch.c,v 1.334.2.3 2020/01/23 12:17:08 ad Exp $ */ /*- * Copyright (c) 1999, 2000, 2004, 2006, 2007, 2008, 2009, 2019 @@ -69,7 +69,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.334.2.2 2020/01/19 21:08:29 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_synch.c,v 1.334.2.3 2020/01/23 12:17:08 ad Exp $"); #include "opt_kstack.h" #include "opt_dtrace.h" @@ -658,9 +658,6 @@ mi_switch(lwp_t *l) /* We're down to only one lock, so do debug checks. */ LOCKDEBUG_BARRIER(l->l_mutex, 1); - /* Disable spinning on any R/W locks that we hold. */ - rw_switch(); - /* Count the context switch. */ CPU_COUNT(CPU_COUNT_NSWTCH, 1); l->l_ncsw++;
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Wed Jan 22 12:10:46 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_lookup.c
Log Message:
Fast-forward through the namecache was stopping one component too soon when
there was an obstacle, e.g. a mountpoint. The obstacle should be returned
not the parent directory.
To generate a diff of this commit:
cvs rdiff -u -r1.212.4.4 -r1.212.4.5 src/sys/kern/vfs_lookup.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.212.4.4 src/sys/kern/vfs_lookup.c:1.212.4.5
--- src/sys/kern/vfs_lookup.c:1.212.4.4 Sun Jan 19 21:19:25 2020
+++ src/sys/kern/vfs_lookup.c Wed Jan 22 12:10:46 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.212.4.4 2020/01/19 21:19:25 ad Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.212.4.5 2020/01/22 12:10:46 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.4 2020/01/19 21:19:25 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.5 2020/01/22 12:10:46 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -954,7 +954,11 @@ lookup_crossmount(struct namei_state *st
(mp = foundobj->v_mountedhere) != NULL &&
(cnp->cn_flags & NOCROSSMOUNT) == 0) {
KASSERTMSG(searchdir != foundobj, "same vn %p", searchdir);
- /* First get the vnode stable. */
+ /*
+ * First get the vnode stable. LK_SHARED works brilliantly
+ * here because almost nothing else wants to lock the
+ * covered vnode.
+ */
error = vn_lock(foundobj, LK_SHARED);
if (error != 0) {
vrele(foundobj);
@@ -962,7 +966,7 @@ lookup_crossmount(struct namei_state *st
break;
}
- /* Then check to see if something is still mounted there. */
+ /* Then check to see if something is still mounted on it. */
if ((mp = foundobj->v_mountedhere) == NULL) {
VOP_UNLOCK(foundobj);
break;
@@ -1109,7 +1113,9 @@ lookup_once(struct namei_state *state,
/*
* If the file system supports VOP_LOOKUP() with a shared lock, and
* we are not making any modifications (nameiop LOOKUP) or this is
- * not the last component then get a shared lock LK_SHARED.
+ * not the last component then get a shared lock LK_SHARED. Where
+ * we can't do fast-forwarded lookups (for example with layered file
+ * systems) then this is the fallback for reducing lock contention.
*/
if ((searchdir->v_mount->mnt_iflag & IMNT_SHRLOOKUP) != 0 &&
(cnp->cn_nameiop == LOOKUP || (cnp->cn_flags & ISLASTCN) == 0)) {
@@ -1237,8 +1243,8 @@ done:
/*
* Parse out the first path name component that we need to to consider.
*
- * While doing this, attempt to use the name cache to fastforward through as
- * many "easy" to find components of the path as possible.
+ * While doing this, attempt to use the name cache to fast-forward through
+ * as many "easy" to find components of the path as possible.
*
* We use the namecache's node locks to form a chain, and avoid as many
* vnode references and locks as possible. In the ideal case, only the
@@ -1280,8 +1286,7 @@ lookup_fastforward(struct namei_state *s
/*
* Can't deal with dotdot lookups, because it means lock
* order reversal, and there are checks in lookup_once()
- * that need to be made. Also check for missing mountpoints
- * (XXX racy).
+ * that need to be made. Also check for missing mountpoints.
*/
if ((cnp->cn_flags & ISDOTDOT) != 0 ||
(*searchdir)->v_mount == NULL) {
@@ -1322,8 +1327,16 @@ lookup_fastforward(struct namei_state *s
break;
}
- /* Stop if we've reached the last component: get vnode. */
- if (cnp->cn_flags & ISLASTCN) {
+ /*
+ * Stop and get a hold on the vnode if there's something
+ * that can't be handled here:
+ *
+ * - we've reached the last component.
+ * - or encountered a mount point that needs to be crossed.
+ * - or encountered something other than a directory.
+ */
+ if ((cnp->cn_flags & ISLASTCN) != 0 || vp->v_type != VDIR ||
+ (vp->v_type == VDIR && vp->v_mountedhere != NULL)) {
mutex_enter(vp->v_interlock);
error = vcache_tryvget(vp);
/* v_interlock now released */
@@ -1335,17 +1348,6 @@ lookup_fastforward(struct namei_state *s
}
/*
- * Not the last component. If we found something other than
- * a directory, or it's a directory with a filesystem
- * mounted on it, bail out.
- */
- if (vp->v_type != VDIR || vp->v_mountedhere != NULL) {
- error = EOPNOTSUPP;
- vp = NULL;
- break;
- }
-
- /*
* Otherwise, we're still in business. Set the found VDIR
* vnode as the search dir for the next component and
* continue on to it.
@@ -1365,9 +1367,8 @@ lookup_fastforward(struct namei_state *s
mutex_enter((*searchdir)->v_interlock);
error2 = vcache_tryvget(*searchdir);
/* v_interlock now unheld */
- if (pl
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sun Jan 19 21:23:36 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_syscalls.c vfs_vnops.c
Log Message:
Use LOCKLEAF in the few cases it's useful for ffs/tmpfs/nullfs. Others need
to be checked.
To generate a diff of this commit:
cvs rdiff -u -r1.539.2.1 -r1.539.2.2 src/sys/kern/vfs_syscalls.c
cvs rdiff -u -r1.204.2.1 -r1.204.2.2 src/sys/kern/vfs_vnops.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_syscalls.c
diff -u src/sys/kern/vfs_syscalls.c:1.539.2.1 src/sys/kern/vfs_syscalls.c:1.539.2.2
--- src/sys/kern/vfs_syscalls.c:1.539.2.1 Fri Jan 17 21:47:35 2020
+++ src/sys/kern/vfs_syscalls.c Sun Jan 19 21:23:36 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_syscalls.c,v 1.539.2.1 2020/01/17 21:47:35 ad Exp $ */
+/* $NetBSD: vfs_syscalls.c,v 1.539.2.2 2020/01/19 21:23:36 ad Exp $ */
/*-
* Copyright (c) 2008, 2009, 2019 The NetBSD Foundation, Inc.
@@ -70,7 +70,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_syscalls.c,v 1.539.2.1 2020/01/17 21:47:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_syscalls.c,v 1.539.2.2 2020/01/19 21:23:36 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_fileassoc.h"
@@ -1532,7 +1532,7 @@ chdir_lookup(const char *path, int where
if (error) {
return error;
}
- NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb);
+ NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | LOCKSHARED | TRYEMULROOT, pb);
if ((error = namei(&nd)) != 0) {
pathbuf_destroy(pb);
return error;
@@ -2996,7 +2996,7 @@ do_sys_accessat(struct lwp *l, int fdat,
return EINVAL;
}
- nd_flag = FOLLOW | LOCKLEAF | TRYEMULROOT;
+ nd_flag = FOLLOW | LOCKLEAF | LOCKSHARED | TRYEMULROOT;
if (flags & AT_SYMLINK_NOFOLLOW)
nd_flag &= ~FOLLOW;
@@ -3222,7 +3222,7 @@ do_sys_readlinkat(struct lwp *l, int fda
if (error) {
return error;
}
- NDINIT(&nd, LOOKUP, NOFOLLOW | LOCKLEAF | TRYEMULROOT, pb);
+ NDINIT(&nd, LOOKUP, NOFOLLOW | LOCKLEAF | LOCKSHARED | TRYEMULROOT, pb);
if ((error = fd_nameiat(l, fdat, &nd)) != 0) {
pathbuf_destroy(pb);
return error;
Index: src/sys/kern/vfs_vnops.c
diff -u src/sys/kern/vfs_vnops.c:1.204.2.1 src/sys/kern/vfs_vnops.c:1.204.2.2
--- src/sys/kern/vfs_vnops.c:1.204.2.1 Fri Jan 17 21:47:35 2020
+++ src/sys/kern/vfs_vnops.c Sun Jan 19 21:23:36 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnops.c,v 1.204.2.1 2020/01/17 21:47:35 ad Exp $ */
+/* $NetBSD: vfs_vnops.c,v 1.204.2.2 2020/01/19 21:23:36 ad Exp $ */
/*-
* Copyright (c) 2009 The NetBSD Foundation, Inc.
@@ -66,7 +66,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.204.2.1 2020/01/17 21:47:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnops.c,v 1.204.2.2 2020/01/19 21:23:36 ad Exp $");
#include "veriexec.h"
@@ -163,7 +163,10 @@ vn_open(struct nameidata *ndp, int fmode
ndp->ni_cnd.cn_flags |= FOLLOW;
} else {
ndp->ni_cnd.cn_nameiop = LOOKUP;
- ndp->ni_cnd.cn_flags |= LOCKLEAF;
+ if ((fmode & O_TRUNC) == 0) /* XXXAD check nfs etc */
+ ndp->ni_cnd.cn_flags |= LOCKLEAF | LOCKSHARED;
+ else
+ ndp->ni_cnd.cn_flags |= LOCKLEAF;
if ((fmode & O_NOFOLLOW) == 0)
ndp->ni_cnd.cn_flags |= FOLLOW;
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sat Jan 18 17:16:20 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
Fix namecache_print().
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.7 -r1.126.2.8 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.7 src/sys/kern/vfs_cache.c:1.126.2.8
--- src/sys/kern/vfs_cache.c:1.126.2.7 Sat Jan 18 15:42:02 2020
+++ src/sys/kern/vfs_cache.c Sat Jan 18 17:16:20 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.7 2020/01/18 15:42:02 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.8 2020/01/18 17:16:20 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -148,7 +148,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.7 2020/01/18 15:42:02 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.8 2020/01/18 17:16:20 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -1388,7 +1388,6 @@ cache_stat_sysctl(SYSCTLFN_ARGS)
void
namecache_print(struct vnode *vp, void (*pr)(const char *, ...))
{
- struct vnode *dvp = NULL;
struct namecache *nc;
struct nchnode *dnn;
enum cache_lru_id id;
@@ -1402,7 +1401,7 @@ namecache_print(struct vnode *vp, void (
}
}
}
- if (dvp == NULL) {
+ if (dnn == NULL) {
(*pr)("name not found\n");
return;
}
CVS commit: [ad-namecache] src/sys/kern
Module Name:src Committed By: ad Date: Sat Jan 18 17:14:13 UTC 2020 Modified Files: src/sys/kern [ad-namecache]: vnode_if.c Log Message: Regen. To generate a diff of this commit: cvs rdiff -u -r1.109 -r1.109.2.1 src/sys/kern/vnode_if.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: src/sys/kern/vnode_if.c diff -u src/sys/kern/vnode_if.c:1.109 src/sys/kern/vnode_if.c:1.109.2.1 --- src/sys/kern/vnode_if.c:1.109 Sun Dec 1 13:58:52 2019 +++ src/sys/kern/vnode_if.c Sat Jan 18 17:14:13 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: vnode_if.c,v 1.109 2019/12/01 13:58:52 ad Exp $ */ +/* $NetBSD: vnode_if.c,v 1.109.2.1 2020/01/18 17:14:13 ad Exp $ */ /* * Warning: DO NOT EDIT! This file is automatically generated! @@ -7,7 +7,7 @@ * Created from the file: * NetBSD: vnode_if.src,v 1.78 2019/10/11 08:04:52 hannken Exp * by the script: - * NetBSD: vnode_if.sh,v 1.68 2019/12/01 13:56:29 ad Exp + * NetBSD: vnode_if.sh,v 1.68.2.1 2020/01/18 17:12:59 ad Exp */ /* @@ -40,7 +40,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: vnode_if.c,v 1.109 2019/12/01 13:58:52 ad Exp $"); +__KERNEL_RCSID(0, "$NetBSD: vnode_if.c,v 1.109.2.1 2020/01/18 17:14:13 ad Exp $"); #include #include @@ -1280,7 +1280,7 @@ VOP_LOCK(struct vnode *vp, a.a_desc = VDESC(vop_lock); a.a_vp = vp; a.a_flags = flags; - error = vop_pre(vp, &mp, &mpsafe, (flags & (LK_UPGRADE|LK_DOWNGRADE) ? FST_NO : (flags & LK_NOWAIT ? FST_TRY : FST_YES))); + error = vop_pre(vp, &mp, &mpsafe, (!(flags & (LK_SHARED|LK_EXCLUSIVE)) ? FST_NO : (flags & LK_NOWAIT ? FST_TRY : FST_YES))); if (error) return error; error = (VCALL(vp, VOFFSET(vop_lock), &a));
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Sat Jan 18 15:42:02 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
- Adjust comments & assertions.
- Ditch memcmp().
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.6 -r1.126.2.7 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.6 src/sys/kern/vfs_cache.c:1.126.2.7
--- src/sys/kern/vfs_cache.c:1.126.2.6 Fri Jan 17 22:26:25 2020
+++ src/sys/kern/vfs_cache.c Sat Jan 18 15:42:02 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.6 2020/01/17 22:26:25 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.7 2020/01/18 15:42:02 ad Exp $ */
/*-
* Copyright (c) 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -148,7 +148,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.6 2020/01/17 22:26:25 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.7 2020/01/18 15:42:02 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -330,6 +330,22 @@ cache_key(const char *name, size_t nlen)
}
/*
+ * Like memcmp() but tuned for small strings of equal length.
+ */
+static inline int
+cache_namecmp(struct namecache *nc, const char *name, size_t namelen)
+{
+ size_t i;
+ int d;
+
+ KASSERT(nc->nc_nlen == namelen);
+ for (d = 0, i = 0; i < namelen; i++) {
+ d |= (nc->nc_name[i] ^ name[i]);
+ }
+ return d;
+}
+
+/*
* Remove an entry from the cache. The directory lock must be held, and if
* "dir2node" is true, then we're locking in the conventional direction and
* the list lock will be acquired when removing the entry from the vnode
@@ -378,11 +394,10 @@ cache_remove(struct namecache *nc, const
}
/*
- * Find a single cache entry and return it locked. The directory lock must
- * be held.
+ * Find a single cache entry and return it. The nchnode lock must be held.
*
* Marked __noinline, and with everything unnecessary excluded, the compiler
- * (gcc 8.3.0 x86_64) does a great job on this.
+ * (gcc 8.3.0 x86_64) does a good job on this.
*/
static struct namecache * __noinline
cache_lookup_entry(struct nchnode *dnn, const char *name, size_t namelen,
@@ -394,12 +409,12 @@ cache_lookup_entry(struct nchnode *dnn,
KASSERT(rw_lock_held(&dnn->nn_lock));
/*
- * Search the RB tree for the key. This is one of the most
- * performance sensitive code paths in the system, so here is an
- * inlined version of rb_tree_find_node() tailored for exactly
- * what's needed here (64-bit key and so on). Elsewhere during
- * entry/removal the generic functions are used as it doesn't
- * matter so much there.
+ * Search the RB tree for the key. File lookup is very performance
+ * sensitive, so here is an inlined lookup tailored for exactly
+ * what's needed here (64-bit key and so on) that is much faster
+ * than using rb_tree_find_node(). Elsewhere during entry/removal
+ * the generic functions are used as it doesn't matter so much
+ * there.
*/
for (;;) {
if (__predict_false(RB_SENTINEL_P(node))) {
@@ -415,8 +430,7 @@ cache_lookup_entry(struct nchnode *dnn,
}
/* Exclude collisions. */
- KASSERT(nc->nc_nlen == namelen);
- if (__predict_false(memcmp(nc->nc_name, name, namelen) != 0)) {
+ if (__predict_false(cache_namecmp(nc, name, namelen))) {
return NULL;
}
@@ -666,7 +680,7 @@ cache_lookup_linked(struct vnode *dvp, c
rw_exit(*plock);
}
*plock = &dnn->nn_lock;
- } else {
+ } else if (*plock == NULL) {
KASSERT(dvp->v_usecount > 0);
}
@@ -674,6 +688,8 @@ cache_lookup_linked(struct vnode *dvp, c
* First up check if the user is allowed to look up files in this
* directory.
*/
+ KASSERT(dnn->nn_mode != VNOVAL && dnn->nn_uid != VNOVAL &&
+ dnn->nn_gid != VNOVAL);
error = kauth_authorize_vnode(cred, KAUTH_ACCESS_ACTION(VEXEC,
dvp->v_type, dnn->nn_mode & ALLPERMS), dvp, NULL,
genfs_can_access(dvp->v_type, dnn->nn_mode & ALLPERMS,
@@ -857,7 +873,7 @@ cache_enter(struct vnode *dvp, struct vn
onc = rb_tree_find_node(&dnn->nn_tree, &nc->nc_key);
if (onc) {
KASSERT(onc->nc_nlen == nc->nc_nlen);
- if (memcmp(onc->nc_name, nc->nc_name, nc->nc_nlen) != 0) {
+ if (cache_namecmp(onc, name, namelen)) {
COUNT(ncs_collisions);
}
cache_remove(onc, true);
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Fri Jan 17 21:54:27 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_getcwd.c
Log Message:
- Get shared locks on the directories, except for VOP_GETATTR() which
unfortunately still needs LK_EXCLUSIVE due to itimes handling in UFS.
- vnode locks are not needed to traverse the namecache.
To generate a diff of this commit:
cvs rdiff -u -r1.53.2.1 -r1.53.2.2 src/sys/kern/vfs_getcwd.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_getcwd.c
diff -u src/sys/kern/vfs_getcwd.c:1.53.2.1 src/sys/kern/vfs_getcwd.c:1.53.2.2
--- src/sys/kern/vfs_getcwd.c:1.53.2.1 Fri Jan 17 21:47:35 2020
+++ src/sys/kern/vfs_getcwd.c Fri Jan 17 21:54:27 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_getcwd.c,v 1.53.2.1 2020/01/17 21:47:35 ad Exp $ */
+/* $NetBSD: vfs_getcwd.c,v 1.53.2.2 2020/01/17 21:54:27 ad Exp $ */
/*-
* Copyright (c) 1999 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_getcwd.c,v 1.53.2.1 2020/01/17 21:47:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_getcwd.c,v 1.53.2.2 2020/01/17 21:54:27 ad Exp $");
#include
#include
@@ -107,6 +107,9 @@ getcwd_scandir(struct vnode **lvpp, stru
int len, reclen;
tries = 0;
+ /* Upgrade to exclusive for UFS VOP_GETATTR (itimes) & VOP_LOOKUP. */
+ vn_lock(lvp, LK_UPGRADE | LK_RETRY);
+
/*
* If we want the filename, get some info we need while the
* current directory is still locked.
@@ -142,7 +145,7 @@ getcwd_scandir(struct vnode **lvpp, stru
}
uvp = *uvpp;
/* Now lvp is unlocked, try to lock uvp */
- error = vn_lock(uvp, LK_EXCLUSIVE);
+ error = vn_lock(uvp, LK_SHARED);
if (error) {
*lvpp = NULL;
*uvpp = NULL;
@@ -254,7 +257,7 @@ unionread:
vput(tvp);
vref(uvp);
*uvpp = uvp;
- vn_lock(uvp, LK_EXCLUSIVE | LK_RETRY);
+ vn_lock(uvp, LK_SHARED | LK_RETRY);
goto unionread;
}
#endif
@@ -310,7 +313,7 @@ getcwd_getcache(struct vnode **lvpp, str
*/
VOP_UNLOCK(lvp);
- vn_lock(uvp, LK_EXCLUSIVE | LK_RETRY);
+ vn_lock(uvp, LK_SHARED | LK_RETRY);
vrele(lvp);
*lvpp = NULL;
@@ -349,7 +352,7 @@ getcwd_common(struct vnode *lvp, struct
* uvp is either NULL, or locked and held.
*/
- vn_lock(lvp, LK_EXCLUSIVE | LK_RETRY);
+ vn_lock(lvp, LK_SHARED | LK_RETRY);
if (bufp)
bp = *bpp;
@@ -396,7 +399,7 @@ getcwd_common(struct vnode *lvp, struct
goto out;
}
vref(lvp);
- error = vn_lock(lvp, LK_EXCLUSIVE | LK_RETRY);
+ error = vn_lock(lvp, LK_SHARED | LK_RETRY);
if (error != 0) {
vrele(lvp);
lvp = NULL;
@@ -557,11 +560,7 @@ vnode_to_path(char *path, size_t len, st
bp = bend = &path[len];
*(--bp) = '\0';
- error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
- if (error != 0)
- return error;
error = cache_revlookup(vp, &dvp, &bp, path);
- VOP_UNLOCK(vp);
if (error != 0)
return (error == -1 ? ENOENT : error);
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Fri Jan 17 21:55:13 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_vnode.c
Log Message:
vrelel: don't change the vnode state to VS_BLOCKED for VOP_INACTIVE(), it's
not needed (at least not for the usual case). Will revist before merge.
To generate a diff of this commit:
cvs rdiff -u -r1.105.2.2 -r1.105.2.3 src/sys/kern/vfs_vnode.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_vnode.c
diff -u src/sys/kern/vfs_vnode.c:1.105.2.2 src/sys/kern/vfs_vnode.c:1.105.2.3
--- src/sys/kern/vfs_vnode.c:1.105.2.2 Fri Jan 17 21:47:35 2020
+++ src/sys/kern/vfs_vnode.c Fri Jan 17 21:55:13 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_vnode.c,v 1.105.2.2 2020/01/17 21:47:35 ad Exp $ */
+/* $NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $ */
/*-
* Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
@@ -119,8 +119,7 @@
* Vnode finished disassociation from underlying file
* system in vcache_reclaim().
* LOADED -> BLOCKED
- * Either vcache_rekey*() is changing the vnode key or
- * vrelel() is about to call VOP_INACTIVE().
+ * vcache_rekey*() is changing the vnode key.
* BLOCKED -> LOADED
* The block condition is over.
* LOADING -> RECLAIMED
@@ -146,7 +145,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.2 2020/01/17 21:47:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $");
#include
#include
@@ -777,24 +776,22 @@ vrelel(vnode_t *vp, int flags, int lktyp
if (VSTATE_GET(vp) == VS_RECLAIMED) {
VOP_UNLOCK(vp);
} else {
- VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
- mutex_exit(vp->v_interlock);
-
/*
- * The vnode must not gain another reference while being
- * deactivated. If VOP_INACTIVE() indicates that
- * the described file has been deleted, then recycle
- * the vnode.
+ * If VOP_INACTIVE() indicates that the described file has
+ * been deleted, then recycle the vnode. Note that
+ * VOP_INACTIVE() will not drop the vnode lock.
*
- * Note that VOP_INACTIVE() will not drop the vnode lock.
+ * If the file has been deleted, this is a lingering
+ * reference and there is no need to worry about new
+ * references looking to do real work with the vnode (as it
+ * will have been purged from directories, caches, etc).
*/
recycle = false;
+ mutex_exit(vp->v_interlock);
VOP_INACTIVE(vp, &recycle);
- if (!recycle)
- VOP_UNLOCK(vp);
mutex_enter(vp->v_interlock);
- VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
if (!recycle) {
+ VOP_UNLOCK(vp);
if (vp->v_usecount > 1) {
vp->v_usecount--;
mutex_exit(vp->v_interlock);
CVS commit: [ad-namecache] src/sys/kern
Module Name:src Committed By: ad Date: Fri Jan 17 21:53:01 UTC 2020 Modified Files: src/sys/kern [ad-namecache]: exec_script.c Log Message: LK_SHARED for VOP_ACCESS(). To generate a diff of this commit: cvs rdiff -u -r1.80 -r1.80.2.1 src/sys/kern/exec_script.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. Modified files: Index: src/sys/kern/exec_script.c diff -u src/sys/kern/exec_script.c:1.80 src/sys/kern/exec_script.c:1.80.2.1 --- src/sys/kern/exec_script.c:1.80 Sun Sep 15 20:21:12 2019 +++ src/sys/kern/exec_script.c Fri Jan 17 21:53:01 2020 @@ -1,4 +1,4 @@ -/* $NetBSD: exec_script.c,v 1.80 2019/09/15 20:21:12 christos Exp $ */ +/* $NetBSD: exec_script.c,v 1.80.2.1 2020/01/17 21:53:01 ad Exp $ */ /* * Copyright (c) 1993, 1994, 1996 Christopher G. Demetriou @@ -31,7 +31,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: exec_script.c,v 1.80 2019/09/15 20:21:12 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: exec_script.c,v 1.80.2.1 2020/01/17 21:53:01 ad Exp $"); #ifdef _KERNEL_OPT #include "opt_script.h" @@ -216,7 +216,7 @@ check_shell: * close all open fd's when the start. That kills this * method of implementing "safe" set-id and x-only scripts. */ - vn_lock(epp->ep_vp, LK_EXCLUSIVE | LK_RETRY); + vn_lock(epp->ep_vp, LK_SHARED | LK_RETRY); error = VOP_ACCESS(epp->ep_vp, VREAD, l->l_cred); VOP_UNLOCK(epp->ep_vp); if (error == EACCES
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Thu Jan 16 16:45:31 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_lookup.c
Log Message:
Push the vnode locking in namei() about as far back as it will go.
To generate a diff of this commit:
cvs rdiff -u -r1.212 -r1.212.4.1 src/sys/kern/vfs_lookup.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_lookup.c
diff -u src/sys/kern/vfs_lookup.c:1.212 src/sys/kern/vfs_lookup.c:1.212.4.1
--- src/sys/kern/vfs_lookup.c:1.212 Thu Jul 18 09:39:40 2019
+++ src/sys/kern/vfs_lookup.c Thu Jan 16 16:45:30 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_lookup.c,v 1.212 2019/07/18 09:39:40 hannken Exp $ */
+/* $NetBSD: vfs_lookup.c,v 1.212.4.1 2020/01/16 16:45:30 ad Exp $ */
/*
* Copyright (c) 1982, 1986, 1989, 1993
@@ -37,7 +37,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212 2019/07/18 09:39:40 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_lookup.c,v 1.212.4.1 2020/01/16 16:45:30 ad Exp $");
#ifdef _KERNEL_OPT
#include "opt_magiclinks.h"
@@ -710,8 +710,6 @@ namei_start(struct namei_state *state, i
return ENOTDIR;
}
- vn_lock(startdir, LK_EXCLUSIVE | LK_RETRY);
-
*startdir_ret = startdir;
return 0;
}
@@ -749,15 +747,17 @@ namei_follow(struct namei_state *state,
size_t linklen;
int error;
- KASSERT(VOP_ISLOCKED(searchdir) == LK_EXCLUSIVE);
- KASSERT(VOP_ISLOCKED(foundobj) == LK_EXCLUSIVE);
if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
return ELOOP;
}
+
+ vn_lock(foundobj, LK_EXCLUSIVE | LK_RETRY);
if (foundobj->v_mount->mnt_flag & MNT_SYMPERM) {
error = VOP_ACCESS(foundobj, VEXEC, cnp->cn_cred);
- if (error != 0)
+ if (error != 0) {
+ VOP_UNLOCK(foundobj);
return error;
+ }
}
/* FUTURE: fix this to not use a second buffer */
@@ -771,6 +771,7 @@ namei_follow(struct namei_state *state,
auio.uio_resid = MAXPATHLEN;
UIO_SETUP_SYSSPACE(&auio);
error = VOP_READLINK(foundobj, &auio, cnp->cn_cred);
+ VOP_UNLOCK(foundobj);
if (error) {
PNBUF_PUT(cp);
return error;
@@ -807,14 +808,11 @@ namei_follow(struct namei_state *state,
/* we're now starting from the beginning of the buffer again */
cnp->cn_nameptr = ndp->ni_pnbuf;
- /* must unlock this before relocking searchdir */
- VOP_UNLOCK(foundobj);
-
/*
* Check if root directory should replace current directory.
*/
if (ndp->ni_pnbuf[0] == '/') {
- vput(searchdir);
+ vrele(searchdir);
/* Keep absolute symbolic links inside emulation root */
searchdir = ndp->ni_erootdir;
if (searchdir == NULL ||
@@ -825,7 +823,6 @@ namei_follow(struct namei_state *state,
searchdir = ndp->ni_rootdir;
}
vref(searchdir);
- vn_lock(searchdir, LK_EXCLUSIVE | LK_RETRY);
while (cnp->cn_nameptr[0] == '/') {
cnp->cn_nameptr++;
ndp->ni_pathlen--;
@@ -833,7 +830,6 @@ namei_follow(struct namei_state *state,
}
*newsearchdir_ret = searchdir;
- KASSERT(VOP_ISLOCKED(searchdir) == LK_EXCLUSIVE);
return 0;
}
@@ -933,19 +929,20 @@ static int
lookup_once(struct namei_state *state,
struct vnode *searchdir,
struct vnode **newsearchdir_ret,
- struct vnode **foundobj_ret)
+ struct vnode **foundobj_ret,
+ bool *newsearchdir_locked_ret)
{
struct vnode *tmpvn; /* scratch vnode */
struct vnode *foundobj; /* result */
struct mount *mp; /* mount table entry */
struct lwp *l = curlwp;
+ bool searchdir_locked = false;
int error;
struct componentname *cnp = state->cnp;
struct nameidata *ndp = state->ndp;
KASSERT(cnp == &ndp->ni_cnd);
- KASSERT(VOP_ISLOCKED(searchdir) == LK_EXCLUSIVE);
*newsearchdir_ret = searchdir;
/*
@@ -977,9 +974,7 @@ lookup_once(struct namei_state *state,
if (ndp->ni_rootdir != rootvnode) {
int retval;
-VOP_UNLOCK(searchdir);
retval = vn_isunder(searchdir, ndp->ni_rootdir, l);
-vn_lock(searchdir, LK_EXCLUSIVE | LK_RETRY);
if (!retval) {
/* Oops! We got out of jail! */
log(LOG_WARNING,
@@ -988,12 +983,11 @@ lookup_once(struct namei_state *state,
p->p_pid, kauth_cred_geteuid(l->l_cred),
p->p_comm);
/* Put us at the jail root. */
-vput(searchdir);
+vrele(searchdir);
searchdir = NULL;
foundobj = ndp->ni_rootdir;
vref(foundobj);
vref(foundobj);
-vn_lock(foundobj, LK_EXCLUSIVE | LK_RETRY);
*newsearchdir_ret = foundobj;
*foundobj_ret = foundobj;
error = 0;
@@ -1006,18 +1000,19 @@ lookup_once(struct namei_state *state,
tmpvn = searchdir;
searchdir = searchdir->v_mount->mnt_vnodecovered;
vref(searchdir);
- vput(tmpvn);
- vn_lock(searchdir, LK_EXCLUSIVE | LK_RETRY);
+ vrele(tmpvn);
*newsearchdir_ret = searchdir;
}
}
/*
* We now have a segment name to search for, and a directory to search.
- * Our vnode state here is that "searchdir" is held and locked.
+ * Our vn
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Wed Jan 8 21:55:10 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
Fix a comment.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.2 -r1.126.2.3 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.2 src/sys/kern/vfs_cache.c:1.126.2.3
--- src/sys/kern/vfs_cache.c:1.126.2.2 Wed Jan 8 11:44:30 2020
+++ src/sys/kern/vfs_cache.c Wed Jan 8 21:55:10 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.2 2020/01/08 11:44:30 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.3 2020/01/08 21:55:10 ad Exp $ */
/*-
* Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.
@@ -149,7 +149,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.2 2020/01/08 11:44:30 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.3 2020/01/08 21:55:10 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -333,7 +333,7 @@ cache_lookup_entry(struct vnode *dvp, co
KASSERT(ncp->nc_dvp == dvp);
/*
* Avoid false sharing: don't write back to nc_hittime
- * unless it has changed within the last 32 ticks.
+ * unless changed significantly.
*/
if (((ncp->nc_hittime ^ hardclock_ticks) & ~31) != 0) {
ncp->nc_hittime = hardclock_ticks;
CVS commit: [ad-namecache] src/sys/kern
Module Name:src
Committed By: ad
Date: Wed Jan 8 11:44:30 UTC 2020
Modified Files:
src/sys/kern [ad-namecache]: vfs_cache.c
Log Message:
cache_enter(): check namelen before touching name.
To generate a diff of this commit:
cvs rdiff -u -r1.126.2.1 -r1.126.2.2 src/sys/kern/vfs_cache.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/vfs_cache.c
diff -u src/sys/kern/vfs_cache.c:1.126.2.1 src/sys/kern/vfs_cache.c:1.126.2.2
--- src/sys/kern/vfs_cache.c:1.126.2.1 Wed Jan 8 11:02:16 2020
+++ src/sys/kern/vfs_cache.c Wed Jan 8 11:44:30 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: vfs_cache.c,v 1.126.2.1 2020/01/08 11:02:16 ad Exp $ */
+/* $NetBSD: vfs_cache.c,v 1.126.2.2 2020/01/08 11:44:30 ad Exp $ */
/*-
* Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.
@@ -149,7 +149,7 @@
*/
#include
-__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.1 2020/01/08 11:02:16 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vfs_cache.c,v 1.126.2.2 2020/01/08 11:44:30 ad Exp $");
#define __NAMECACHE_PRIVATE
#ifdef _KERNEL_OPT
@@ -730,8 +730,8 @@ cache_enter(struct vnode *dvp, struct vn
ncp->nc_whiteout = false;
vi = VNODE_TO_VIMPL(vp);
/* Partially sort the per-vnode list: dots go to back. */
- if ((name[0] == '.' && namelen == 1) ||
- (name[0] == '.' && name[1] == '.' && namelen == 2)) {
+ if ((namelen == 1 && name[0] == '.') ||
+ (namelen == 2 && name[0] == '.' && name[1] == '.')) {
TAILQ_INSERT_TAIL(&vi->vi_nclist, ncp, nc_vlist);
} else {
TAILQ_INSERT_HEAD(&vi->vi_nclist, ncp, nc_vlist);
