On 9 Jul, I wrote:
> The patch below enhances the mutex pool code to support the creation and
> use of multiple mutex pools. It creates one pool of sleep mutexes with
> the MTX_NOWITNESS flag for use in building higher level (sx and lockmgr)
> locks. It also creates another pool without MTX_NOWITNESS for general
> purpose use. It can also be used to create pools of spin mutexes.
>
> The users of the existing MTX_NOWITNESS pool are modified to use the
> appropriate pool.
>
> If this had been implemented earlier, it would have allowed witness to
> catch the Giant vs. FILE_LOCK() lock order reversal that I recently
> tracked down and fixed.
>
> I've tested this only on i386, but it also passes the "make universe"
> test.
It might help to actually include the patch ...
Index: sys/kern/kern_descrip.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_descrip.c,v
retrieving revision 1.207
diff -u -r1.207 kern_descrip.c
--- sys/kern/kern_descrip.c 4 Jul 2003 12:20:27 -0000 1.207
+++ sys/kern/kern_descrip.c 5 Jul 2003 22:21:42 -0000
@@ -1209,7 +1209,7 @@
* descriptor to the list of open files at that point, otherwise
* put it at the front of the list of open files.
*/
- fp->f_mtxp = mtx_pool_alloc();
+ fp->f_mtxp = mtx_pool_alloc(mtxpool_sleep);
fp->f_count = 1;
fp->f_cred = crhold(td->td_ucred);
fp->f_ops = &badfileops;
Index: sys/kern/kern_lock.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_lock.c,v
retrieving revision 1.68
diff -u -r1.68 kern_lock.c
--- sys/kern/kern_lock.c 11 Jun 2003 00:56:56 -0000 1.68
+++ sys/kern/kern_lock.c 5 Jul 2003 20:28:33 -0000
@@ -547,9 +547,9 @@
* XXX cleanup - make sure mtxpool is always initialized before
* this is ever called.
*/
- if (mtx_pool_valid) {
+ if (mtxpool_lockbuilder != NULL) {
mtx_lock(&lock_mtx);
- lkp->lk_interlock = mtx_pool_alloc();
+ lkp->lk_interlock = mtx_pool_alloc(mtxpool_lockbuilder);
mtx_unlock(&lock_mtx);
} else {
lkp->lk_interlock = &lock_mtx;
Index: sys/kern/kern_mtxpool.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_mtxpool.c,v
retrieving revision 1.7
diff -u -r1.7 kern_mtxpool.c
--- sys/kern/kern_mtxpool.c 13 Jun 2003 19:39:21 -0000 1.7
+++ sys/kern/kern_mtxpool.c 9 Jul 2003 05:10:51 -0000
@@ -35,85 +35,164 @@
#include <sys/mutex.h>
#include <sys/systm.h>
-#ifndef MTX_POOL_SIZE
-#define MTX_POOL_SIZE 128
-#endif
-#define MTX_POOL_MASK (MTX_POOL_SIZE - 1)
-static struct mtx mtx_pool_ary[MTX_POOL_SIZE];
+MALLOC_DEFINE(M_MTXPOOL, "mtx_pool", "mutex pool");
-int mtx_pool_valid = 0;
+/* Pool sizes must be a power of two */
+#ifndef MTX_POOL_LOCKBUILDER_SIZE
+#define MTX_POOL_LOCKBUILDER_SIZE 128
+#endif
+#ifndef MTX_POOL_SLEEP_SIZE
+#define MTX_POOL_SLEEP_SIZE 128
+#endif
+
+struct mtxpool_header {
+ int mtxpool_size;
+ int mtxpool_mask;
+ int mtxpool_shift;
+ int mtxpool_next;
+};
+
+struct mtx_pool {
+ struct mtxpool_header mtx_pool_header;
+ struct mtx mtx_pool_ary[1];
+};
+
+static struct mtx_pool_lockbuilder {
+ struct mtxpool_header mtx_pool_header;
+ struct mtx mtx_pool_ary[MTX_POOL_LOCKBUILDER_SIZE];
+} lockbuilder_pool;
+
+#define mtx_pool_size mtx_pool_header.mtxpool_size
+#define mtx_pool_mask mtx_pool_header.mtxpool_mask
+#define mtx_pool_shift mtx_pool_header.mtxpool_shift
+#define mtx_pool_next mtx_pool_header.mtxpool_next
+
+struct mtx_pool *mtxpool_sleep;
+struct mtx_pool *mtxpool_lockbuilder;
+
+#if UINTPTR_MAX == UINT64_MAX /* 64 bits */
+# define POINTER_BITS 64
+# define HASH_MULTIPLIER 11400714819323198485u /* (2^64)*(sqrt(5)-1)/2 */
+#else /* assume 32 bits */
+# define POINTER_BITS 32
+# define HASH_MULTIPLIER 2654435769u /* (2^32)*(sqrt(5)-1)/2 */
+#endif
/*
- * Inline version of mtx_pool_find(), used to streamline our main API
- * function calls.
+ * Return the (shared) pool mutex associated with the specified address.
+ * The returned mutex is a leaf level mutex, meaning that if you obtain it
+ * you cannot obtain any other mutexes until you release it. You can
+ * legally msleep() on the mutex.
*/
-static __inline struct mtx *
-_mtx_pool_find(void *ptr)
+struct mtx *
+mtx_pool_find(struct mtx_pool *pool, void *ptr)
{
- int p;
+ int p;
- p = (int)(uintptr_t)ptr;
- return (&mtx_pool_ary[(p ^ (p >> 6)) & MTX_POOL_MASK]);
+ KASSERT(pool != NULL, ("_mtx_pool_find(): null pool"));
+ /*
+ * Fibonacci hash, see Knuth's
+ * _Art of Computer Programming, Volume 3 / Sorting and Searching_
+ */
+ p = ((HASH_MULTIPLIER * (uintptr_t)ptr) >> pool->mtx_pool_shift) &
+ pool->mtx_pool_mask;
+ return (&pool->mtx_pool_ary[p]);
}
static void
-mtx_pool_setup(void *dummy __unused)
+mtx_pool_initialize(struct mtx_pool *pool, const char *mtx_name, int pool_size,
+ int opts)
{
- int i;
+ int i, maskbits;
- for (i = 0; i < MTX_POOL_SIZE; ++i)
- mtx_init(&mtx_pool_ary[i], "pool mutex", NULL,
- MTX_DEF | MTX_NOWITNESS | MTX_QUIET);
- mtx_pool_valid = 1;
+ pool->mtx_pool_size = pool_size;
+ pool->mtx_pool_mask = pool_size - 1;
+ for (i = 1, maskbits = 0; (i & pool_size) == 0; i = i << 1)
+ maskbits++;
+ pool->mtx_pool_shift = POINTER_BITS - maskbits;
+ pool->mtx_pool_next = 0;
+ for (i = 0; i < pool_size; ++i)
+ mtx_init(&pool->mtx_pool_ary[i], mtx_name, NULL, opts);
+}
+
+struct mtx_pool *
+mtx_pool_create(const char *mtx_name, int pool_size, int opts)
+{
+ struct mtx_pool *pool;
+
+ if (pool_size <= 0 || !powerof2(pool_size)) {
+ printf("WARNING: %s pool size is not a power of 2.\n",
+ mtx_name);
+ pool_size = 128;
+ }
+ MALLOC(pool, struct mtx_pool *,
+ sizeof (struct mtx_pool) + ((pool_size - 1) * sizeof (struct mtx)),
+ M_MTXPOOL, M_WAITOK | M_ZERO);
+ mtx_pool_initialize(pool, mtx_name, pool_size, opts);
+ return pool;
}
-/*
- * Obtain a (shared) mutex from the pool. The returned mutex is a leaf
- * level mutex, meaning that if you obtain it you cannot obtain any other
- * mutexes until you release it. You can legally msleep() on the mutex.
- */
-struct mtx *
-mtx_pool_alloc(void)
+void
+mtx_pool_destroy(struct mtx_pool **poolp)
{
- static int si;
+ int i;
+ struct mtx_pool *pool = *poolp;
- return (&mtx_pool_ary[si++ & MTX_POOL_MASK]);
+ for (i = pool->mtx_pool_size - 1; i >= 0; --i)
+ mtx_destroy(&pool->mtx_pool_ary[i]);
+ FREE(pool, M_MTXPOOL);
+ *poolp = NULL;
}
-/*
- * Return the (shared) pool mutex associated with the specified address.
- * The returned mutex is a leaf level mutex, meaning that if you obtain it
- * you cannot obtain any other mutexes until you release it. You can
- * legally msleep() on the mutex.
- */
-struct mtx *
-mtx_pool_find(void *ptr)
+static void
+mtx_pool_setup_static(void *dummy __unused)
{
-
- return (_mtx_pool_find(ptr));
+ mtx_pool_initialize((struct mtx_pool *)&lockbuilder_pool,
+ "lockbuilder mtxpool", MTX_POOL_LOCKBUILDER_SIZE,
+ MTX_DEF | MTX_NOWITNESS | MTX_QUIET);
+ mtxpool_lockbuilder = (struct mtx_pool *)&lockbuilder_pool;
}
-/*
- * Combined find/lock operation. Lock the pool mutex associated with
- * the specified address.
- */
-void
-mtx_pool_lock(void *ptr)
+static void
+mtx_pool_setup_dynamic(void *dummy __unused)
{
-
- mtx_lock(_mtx_pool_find(ptr));
+ mtxpool_sleep = mtx_pool_create("sleep mtxpool",
+ MTX_POOL_SLEEP_SIZE, MTX_DEF);
}
/*
- * Combined find/unlock operation. Unlock the pool mutex associated with
- * the specified address.
+ * Obtain a (shared) mutex from the pool. The returned mutex is a leaf
+ * level mutex, meaning that if you obtain it you cannot obtain any other
+ * mutexes until you release it. You can legally msleep() on the mutex.
*/
-void
-mtx_pool_unlock(void *ptr)
+struct mtx *
+mtx_pool_alloc(struct mtx_pool *pool)
{
+ int i;
- mtx_unlock(_mtx_pool_find(ptr));
+ KASSERT(pool != NULL, ("mtx_pool_alloc(): null pool"));
+ /*
+ * mtx_pool_next is unprotected against multiple accesses,
+ * but simultaneous access by two CPUs should not be very
+ * harmful.
+ */
+ i = pool->mtx_pool_next;
+ pool->mtx_pool_next = (i + 1) & pool->mtx_pool_mask;
+ return (&pool->mtx_pool_ary[i]);
}
-SYSINIT(mtxpooli, SI_SUB_MTX_POOL, SI_ORDER_FIRST, mtx_pool_setup, NULL);
+/*
+ * The lockbuilder pool must be initialized early because the lockmgr
+ * and sx locks depend on it. The sx locks are used in the kernel
+ * memory allocator. The lockmgr subsystem is initialized by
+ * SYSINIT(..., SI_SUB_LOCK, ...).
+ *
+ * We can't call MALLOC() to dynamically allocate the sleep pool
+ * until after kmeminit() has been called, which is done by
+ * SYSINIT(..., SI_SUB_KMEM, ...).
+ */
+SYSINIT(mtxpooli1, SI_SUB_MTX_POOL_STATIC, SI_ORDER_FIRST,
+ mtx_pool_setup_static, NULL);
+SYSINIT(mtxpooli2, SI_SUB_MTX_POOL_DYNAMIC, SI_ORDER_FIRST,
+ mtx_pool_setup_dynamic, NULL);
Index: sys/kern/kern_prot.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_prot.c,v
retrieving revision 1.174
diff -u -r1.174 kern_prot.c
--- sys/kern/kern_prot.c 4 Jul 2003 02:21:28 -0000 1.174
+++ sys/kern/kern_prot.c 5 Jul 2003 22:21:42 -0000
@@ -1652,7 +1652,7 @@
MALLOC(cr, struct ucred *, sizeof(*cr), M_CRED, M_WAITOK | M_ZERO);
cr->cr_ref = 1;
- cr->cr_mtxp = mtx_pool_find(cr);
+ cr->cr_mtxp = mtx_pool_find(mtxpool_sleep, cr);
#ifdef MAC
mac_init_cred(cr);
#endif
Index: sys/kern/kern_resource.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_resource.c,v
retrieving revision 1.126
diff -u -r1.126 kern_resource.c
--- sys/kern/kern_resource.c 11 Jun 2003 00:56:56 -0000 1.126
+++ sys/kern/kern_resource.c 5 Jul 2003 03:21:48 -0000
@@ -893,7 +893,7 @@
free(uip, M_UIDINFO);
uip = old_uip;
} else {
- uip->ui_mtxp = mtx_pool_alloc();
+ uip->ui_mtxp = mtx_pool_alloc(mtxpool_sleep);
uip->ui_uid = uid;
LIST_INSERT_HEAD(UIHASH(uid), uip, ui_hash);
}
Index: sys/kern/kern_sx.c
===================================================================
RCS file: /home/ncvs/src/sys/kern/kern_sx.c,v
retrieving revision 1.19
diff -u -r1.19 kern_sx.c
--- sys/kern/kern_sx.c 11 Jun 2003 00:56:56 -0000 1.19
+++ sys/kern/kern_sx.c 5 Jul 2003 03:22:11 -0000
@@ -74,7 +74,7 @@
lock->lo_type = lock->lo_name = description;
lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE |
LO_UPGRADABLE;
- sx->sx_lock = mtx_pool_find(sx);
+ sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx);
sx->sx_cnt = 0;
cv_init(&sx->sx_shrd_cv, description);
sx->sx_shrd_wcnt = 0;
Index: sys/sys/kernel.h
===================================================================
RCS file: /home/ncvs/src/sys/sys/kernel.h,v
retrieving revision 1.110
diff -u -r1.110 kernel.h
--- sys/sys/kernel.h 3 Jun 2003 08:41:04 -0000 1.110
+++ sys/sys/kernel.h 5 Jul 2003 20:47:40 -0000
@@ -112,11 +112,12 @@
SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */
SI_SUB_CONSOLE = 0x0800000, /* console*/
SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/
- SI_SUB_MTX_POOL = 0x0900000, /* mutex pool */
+ SI_SUB_MTX_POOL_STATIC = 0x0900000, /* static mutex pool */
SI_SUB_VM = 0x1000000, /* virtual memory system init*/
SI_SUB_KMEM = 0x1800000, /* kernel memory*/
SI_SUB_KVM_RSRC = 0x1A00000, /* kvm operational limits*/
SI_SUB_WITNESS = 0x1A80000, /* witness initialization */
+ SI_SUB_MTX_POOL_DYNAMIC = 0x1AC0000, /* dynamic mutex pool */
SI_SUB_LOCK = 0x1B00000, /* lockmgr locks */
SI_SUB_EVENTHANDLER = 0x1C00000, /* eventhandler init */
SI_SUB_KLD = 0x2000000, /* KLD and module setup */
Index: sys/sys/mutex.h
===================================================================
RCS file: /home/ncvs/src/sys/sys/mutex.h,v
retrieving revision 1.61
diff -u -r1.61 mutex.h
--- sys/sys/mutex.h 18 May 2003 03:46:30 -0000 1.61
+++ sys/sys/mutex.h 9 Jul 2003 02:04:05 -0000
@@ -236,12 +236,30 @@
#define mtx_unlock(m) mtx_unlock_flags((m), 0)
#define mtx_unlock_spin(m) mtx_unlock_spin_flags((m), 0)
-struct mtx *mtx_pool_find(void *ptr);
-struct mtx *mtx_pool_alloc(void);
-void mtx_pool_lock(void *ptr);
-void mtx_pool_unlock(void *ptr);
+struct mtx_pool;
-extern int mtx_pool_valid;
+struct mtx_pool *mtx_pool_create(const char *mtx_name, int pool_size, int opts);
+void mtx_pool_destroy(struct mtx_pool **poolp);
+struct mtx *mtx_pool_find(struct mtx_pool *pool, void *ptr);
+struct mtx *mtx_pool_alloc(struct mtx_pool *pool);
+struct mtx *mtx_pool_alloc_spin(struct mtx_pool *pool);
+#define mtx_pool_lock(pool, ptr) \
+ mtx_lock(mtx_pool_find((pool), (ptr)))
+#define mtx_pool_lock_spin(pool, ptr) \
+ mtx_lock_spin(mtx_pool_find((pool), (ptr)))
+#define mtx_pool_unlock(pool, ptr) \
+ mtx_unlock(mtx_pool_find((pool), (ptr)))
+#define mtx_pool_unlock_spin(pool, ptr) \
+ mtx_unlock_spin(mtx_pool_find((pool), (ptr)))
+
+/*
+ * mtxpool_lockbuilder is a pool of sleep locks that is not witness
+ * checked and should only be used for building higher level locks.
+ *
+ * mtxpool_sleep is a general purpose pool of sleep mutexes.
+ */
+extern struct mtx_pool *mtxpool_lockbuilder;
+extern struct mtx_pool *mtxpool_sleep;
#ifndef LOCK_DEBUG
#error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/mutex.h>
_______________________________________________
[EMAIL PROTECTED] mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-current
To unsubscribe, send any mail to "[EMAIL PROTECTED]"
