Module Name: src
Committed By: ad
Date: Sat May 23 18:28:06 UTC 2009
Modified Files:
src/sys/kern: kern_descrip.c kern_lwp.c kern_proc.c
src/sys/sys: filedesc.h
Log Message:
Make descriptor access and file allocation cheaper in many cases,
mostly by avoiding a bunch of atomic operations.
To generate a diff of this commit:
cvs rdiff -u -r1.190 -r1.191 src/sys/kern/kern_descrip.c
cvs rdiff -u -r1.130 -r1.131 src/sys/kern/kern_lwp.c
cvs rdiff -u -r1.151 -r1.152 src/sys/kern/kern_proc.c
cvs rdiff -u -r1.53 -r1.54 src/sys/sys/filedesc.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_descrip.c
diff -u src/sys/kern/kern_descrip.c:1.190 src/sys/kern/kern_descrip.c:1.191
--- src/sys/kern/kern_descrip.c:1.190 Sat Apr 4 10:12:51 2009
+++ src/sys/kern/kern_descrip.c Sat May 23 18:28:05 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_descrip.c,v 1.190 2009/04/04 10:12:51 ad Exp $ */
+/* $NetBSD: kern_descrip.c,v 1.191 2009/05/23 18:28:05 ad Exp $ */
/*-
* Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
@@ -70,7 +70,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_descrip.c,v 1.190 2009/04/04 10:12:51 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_descrip.c,v 1.191 2009/05/23 18:28:05 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -346,16 +346,25 @@
return NULL;
}
- /*
- * Now get a reference to the descriptor. Issue a memory
- * barrier to ensure that we acquire the file pointer _after_
- * adding a reference. If no memory barrier, we could fetch
- * a stale pointer.
- */
- atomic_inc_uint(&ff->ff_refcnt);
+ /* Now get a reference to the descriptor. */
+ if (fdp->fd_refcnt == 1) {
+ /*
+ * Single threaded: don't need to worry about concurrent
+ * access (other than earlier calls to kqueue, which may
+ * hold a reference to the descriptor).
+ */
+ ff->ff_refcnt++;
+ } else {
+ /*
+ * Issue a memory barrier to ensure that we acquire the file
+ * pointer _after_ adding a reference. If no memory
+ * barrier, we could fetch a stale pointer.
+ */
+ atomic_inc_uint(&ff->ff_refcnt);
#ifndef __HAVE_ATOMIC_AS_MEMBAR
- membar_enter();
+ membar_enter();
#endif
+ }
/*
* If the file is not open or is being closed then put the
@@ -387,6 +396,20 @@
KASSERT((ff->ff_refcnt & FR_MASK) > 0);
KASSERT(fd >= NDFDFILE || ff == (fdfile_t *)fdp->fd_dfdfile[fd]);
+ if (fdp->fd_refcnt == 1) {
+ /*
+ * Single threaded: don't need to worry about concurrent
+ * access (other than earlier calls to kqueue, which may
+ * hold a reference to the descriptor).
+ */
+ if (__predict_false((ff->ff_refcnt & FR_CLOSING) != 0)) {
+ fd_close(fd);
+ return;
+ }
+ ff->ff_refcnt--;
+ return;
+ }
+
/*
* Ensure that any use of the file is complete and globally
* visible before dropping the final reference. If no membar,
@@ -747,7 +770,9 @@
} else {
error = 0;
}
- ffree(fp);
+ KASSERT(fp->f_count == 0);
+ KASSERT(fp->f_cred != NULL);
+ pool_cache_put(file_cache, fp);
return error;
}
@@ -761,14 +786,10 @@
filedesc_t *fdp;
int i, lim, last, error;
u_int off, new;
- fdfile_t *ff;
KASSERT(p == curproc || p == &proc0);
fdp = p->p_fd;
- ff = pool_cache_get(fdfile_cache, PR_WAITOK);
- KASSERT(ff->ff_refcnt == 0);
- KASSERT(ff->ff_file == NULL);
/*
* Search for a free descriptor starting at the higher
@@ -802,10 +823,10 @@
}
if (fdp->fd_ofiles[i] == NULL) {
KASSERT(i >= NDFDFILE);
- fdp->fd_ofiles[i] = ff;
- } else {
- pool_cache_put(fdfile_cache, ff);
+ fdp->fd_ofiles[i] =
+ pool_cache_get(fdfile_cache, PR_WAITOK);
}
+ KASSERT(fdp->fd_ofiles[i]->ff_refcnt == 0);
KASSERT(fdp->fd_ofiles[i]->ff_file == NULL);
fd_used(fdp, i);
if (want <= fdp->fd_freefile) {
@@ -821,7 +842,6 @@
/* No space in current array. Let the caller expand and retry. */
error = (fdp->fd_nfiles >= lim) ? EMFILE : ENOSPC;
mutex_exit(&fdp->fd_lock);
- pool_cache_put(fdfile_cache, ff);
return error;
}
@@ -943,7 +963,7 @@
* to other threads.
*/
if (oldnfiles > NDFILE) {
- if ((fdp->fd_refcnt | p->p_nlwps) > 1) {
+ if (fdp->fd_refcnt > 1) {
fdp->fd_ofiles[-2] = (void *)fdp->fd_discard;
fdp->fd_discard = fdp->fd_ofiles - 2;
} else {
@@ -988,6 +1008,7 @@
int
fd_allocfile(file_t **resultfp, int *resultfd)
{
+ kauth_cred_t cred;
file_t *fp;
proc_t *p;
int error;
@@ -1002,29 +1023,32 @@
}
fp = pool_cache_get(file_cache, PR_WAITOK);
+ if (fp == NULL) {
+ return ENFILE;
+ }
KASSERT(fp->f_count == 0);
KASSERT(fp->f_msgcount == 0);
KASSERT(fp->f_unpcount == 0);
- fp->f_cred = kauth_cred_get();
- kauth_cred_hold(fp->f_cred);
- if (__predict_false(atomic_inc_uint_nv(&nfiles) >= maxfiles)) {
- fd_abort(p, fp, *resultfd);
- tablefull("file", "increase kern.maxfiles or MAXFILES");
- return ENFILE;
+ /* Replace cached credentials if not what we need. */
+ cred = curlwp->l_cred;
+ if (__predict_false(cred != fp->f_cred)) {
+ kauth_cred_free(fp->f_cred);
+ kauth_cred_hold(cred);
+ fp->f_cred = cred;
}
/*
* Don't allow recycled files to be scanned.
+ * See uipc_usrreq.c.
*/
- if ((fp->f_flag & FSCAN) != 0) {
+ if (__predict_false((fp->f_flag & FSCAN) != 0)) {
mutex_enter(&fp->f_lock);
atomic_and_uint(&fp->f_flag, ~FSCAN);
mutex_exit(&fp->f_lock);
}
fp->f_advice = 0;
- fp->f_msgcount = 0;
fp->f_offset = 0;
*resultfp = fp;
@@ -1092,34 +1116,30 @@
mutex_exit(&fdp->fd_lock);
if (fp != NULL) {
- ffree(fp);
+ KASSERT(fp->f_count == 0);
+ KASSERT(fp->f_cred != NULL);
+ pool_cache_put(file_cache, fp);
}
}
-/*
- * Free a file descriptor.
- */
-void
-ffree(file_t *fp)
-{
-
- KASSERT(fp->f_count == 0);
-
- atomic_dec_uint(&nfiles);
- kauth_cred_free(fp->f_cred);
- pool_cache_put(file_cache, fp);
-}
-
static int
file_ctor(void *arg, void *obj, int flags)
{
file_t *fp = obj;
memset(fp, 0, sizeof(*fp));
- mutex_init(&fp->f_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_enter(&filelist_lock);
+ if (__predict_false(nfiles >= maxfiles)) {
+ mutex_exit(&filelist_lock);
+ tablefull("file", "increase kern.maxfiles or MAXFILES");
+ return ENFILE;
+ }
+ nfiles++;
LIST_INSERT_HEAD(&filehead, fp, f_list);
+ mutex_init(&fp->f_lock, MUTEX_DEFAULT, IPL_NONE);
+ fp->f_cred = curlwp->l_cred;
+ kauth_cred_hold(fp->f_cred);
mutex_exit(&filelist_lock);
return 0;
@@ -1131,9 +1151,11 @@
file_t *fp = obj;
mutex_enter(&filelist_lock);
+ nfiles--;
LIST_REMOVE(fp, f_list);
mutex_exit(&filelist_lock);
+ kauth_cred_free(fp->f_cred);
mutex_destroy(&fp->f_lock);
}
@@ -1260,6 +1282,16 @@
}
/*
+ * Acquire a hold on a filedesc structure.
+ */
+void
+fd_hold(void)
+{
+
+ atomic_inc_uint(&curlwp->l_fd->fd_refcnt);
+}
+
+/*
* Copy a filedesc structure.
*/
filedesc_t *
Index: src/sys/kern/kern_lwp.c
diff -u src/sys/kern/kern_lwp.c:1.130 src/sys/kern/kern_lwp.c:1.131
--- src/sys/kern/kern_lwp.c:1.130 Sat May 23 18:21:20 2009
+++ src/sys/kern/kern_lwp.c Sat May 23 18:28:06 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_lwp.c,v 1.130 2009/05/23 18:21:20 ad Exp $ */
+/* $NetBSD: kern_lwp.c,v 1.131 2009/05/23 18:28:06 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -210,7 +210,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.130 2009/05/23 18:21:20 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lwp.c,v 1.131 2009/05/23 18:28:06 ad Exp $");
#include "opt_ddb.h"
#include "opt_lockdebug.h"
@@ -236,6 +236,7 @@
#include <sys/intr.h>
#include <sys/lwpctl.h>
#include <sys/atomic.h>
+#include <sys/filedesc.h>
#include <uvm/uvm_extern.h>
#include <uvm/uvm_object.h>
@@ -596,9 +597,20 @@
l2->l_inheritedprio = -1;
l2->l_flag = inmem ? LW_INMEM : 0;
l2->l_pflag = LP_MPSAFE;
- l2->l_fd = p2->p_fd;
TAILQ_INIT(&l2->l_ld_locks);
+ /*
+ * If not the first LWP in the process, grab a reference to the
+ * descriptor table.
+ */
+ l2->l_fd = p2->p_fd;
+ if (p2->p_nlwps != 0) {
+ KASSERT(l1->l_proc == p2);
+ fd_hold();
+ } else {
+ KASSERT(l1->l_proc != p2);
+ }
+
if (p2->p_flag & PK_SYSTEM) {
/* Mark it as a system LWP and not a candidate for swapping */
l2->l_flag |= LW_SYSTEM;
@@ -723,6 +735,7 @@
current = (l == curlwp);
KASSERT(current || (l->l_stat == LSIDL && l->l_target_cpu == NULL));
+ KASSERT(p == curproc);
/*
* Verify that we hold no locks other than the kernel lock.
@@ -752,6 +765,9 @@
if (p->p_emul->e_lwp_exit)
(*p->p_emul->e_lwp_exit)(l);
+ /* Drop filedesc reference. */
+ fd_free();
+
/* Delete the specificdata while it's still safe to sleep. */
specificdata_fini(lwp_specificdata_domain, &l->l_specdataref);
Index: src/sys/kern/kern_proc.c
diff -u src/sys/kern/kern_proc.c:1.151 src/sys/kern/kern_proc.c:1.152
--- src/sys/kern/kern_proc.c:1.151 Sat Apr 25 15:06:31 2009
+++ src/sys/kern/kern_proc.c Sat May 23 18:28:06 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_proc.c,v 1.151 2009/04/25 15:06:31 rmind Exp $ */
+/* $NetBSD: kern_proc.c,v 1.152 2009/05/23 18:28:06 ad Exp $ */
/*-
* Copyright (c) 1999, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.151 2009/04/25 15:06:31 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.152 2009/05/23 18:28:06 ad Exp $");
#include "opt_kstack.h"
#include "opt_maxuprc.h"
@@ -203,6 +203,7 @@
.l_psid = PS_NONE,
.l_pi_lenders = SLIST_HEAD_INITIALIZER(&lwp0.l_pi_lenders),
.l_name = __UNCONST("swapper"),
+ .l_fd = &filedesc0,
};
kauth_cred_t cred0;
Index: src/sys/sys/filedesc.h
diff -u src/sys/sys/filedesc.h:1.53 src/sys/sys/filedesc.h:1.54
--- src/sys/sys/filedesc.h:1.53 Tue Nov 18 11:36:58 2008
+++ src/sys/sys/filedesc.h Sat May 23 18:28:05 2009
@@ -1,4 +1,4 @@
-/* $NetBSD: filedesc.h,v 1.53 2008/11/18 11:36:58 pooka Exp $ */
+/* $NetBSD: filedesc.h,v 1.54 2009/05/23 18:28:05 ad Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc.
@@ -87,7 +87,7 @@
#define NDENTRYSHIFT 5 /* bits per entry */
#define NDLOSLOTS(x) (((x) + NDENTRIES - 1) >> NDENTRYSHIFT)
#define NDHISLOTS(x) ((NDLOSLOTS(x) + NDENTRIES - 1) >> NDENTRYSHIFT)
-#define NDFDFILE 3 /* first 3 descriptors are free */
+#define NDFDFILE 6 /* first 6 descriptors are free */
/*
* Process-private descriptor reference, one for each descriptor slot
@@ -182,6 +182,7 @@
filedesc_t *fd_copy(void);
filedesc_t *fd_init(filedesc_t *);
void fd_share(proc_t *);
+void fd_hold(void);
void fd_clear(void);
void fd_free(void);
void fd_remove(filedesc_t *, unsigned);
@@ -214,7 +215,6 @@
int vnode_to_path(char *, size_t, struct vnode *, struct lwp *,
struct proc *);
-void ffree(file_t *);
int closef(file_t *);
file_t *fgetdummy(void);
void fputdummy(file_t *);
