Module Name: src
Committed By: riastradh
Date: Thu Feb 23 14:57:29 UTC 2023
Modified Files:
src/sys/kern: kern_lock.c kern_mutex.c
Log Message:
KERNEL_LOCK(9): Minor tweaks to ci->ci_biglock_wanted access.
1. Use atomic_load_relaxed to read ci->ci_biglock_wanted from another
CPU, for clarity and to avoid the appearance of data races in thread
sanitizers. (Reading ci->ci_biglock_wanted on the local CPU need
not be atomic because no other CPU can be writing to it.)
2. Use atomic_store_relaxed to update ci->ci_biglock_wanted when we
start to spin, to avoid the appearance of data races.
3. Add comments to explain what's going on and cross-reference the
specific matching membars in mutex_vector_enter.
related to PR kern/57240
To generate a diff of this commit:
cvs rdiff -u -r1.182 -r1.183 src/sys/kern/kern_lock.c
cvs rdiff -u -r1.102 -r1.103 src/sys/kern/kern_mutex.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_lock.c
diff -u src/sys/kern/kern_lock.c:1.182 src/sys/kern/kern_lock.c:1.183
--- src/sys/kern/kern_lock.c:1.182 Fri Jan 27 09:28:41 2023
+++ src/sys/kern/kern_lock.c Thu Feb 23 14:57:29 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_lock.c,v 1.182 2023/01/27 09:28:41 ozaki-r Exp $ */
+/* $NetBSD: kern_lock.c,v 1.183 2023/02/23 14:57:29 riastradh Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.182 2023/01/27 09:28:41 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.183 2023/02/23 14:57:29 riastradh Exp $");
#ifdef _KERNEL_OPT
#include "opt_lockdebug.h"
@@ -235,10 +235,18 @@ _kernel_lock(int nlocks)
* is required to ensure that the result of any mutex_exit()
* by the current LWP becomes visible on the bus before the set
* of ci->ci_biglock_wanted becomes visible.
+ *
+ * This membar_producer matches the membar_consumer in
+ * mutex_vector_enter.
+ *
+ * That way, if l has just released a mutex, mutex_vector_enter
+ * can't see this store ci->ci_biglock_wanted := l until it
+ * will also see the mutex_exit store mtx->mtx_owner := 0 which
+ * clears the has-waiters bit.
*/
membar_producer();
owant = ci->ci_biglock_wanted;
- ci->ci_biglock_wanted = l;
+ atomic_store_relaxed(&ci->ci_biglock_wanted, l);
#if defined(DIAGNOSTIC) && !defined(LOCKDEBUG)
l->l_ld_wanted = __builtin_return_address(0);
#endif
@@ -286,22 +294,20 @@ _kernel_lock(int nlocks)
/*
* Now that we have kernel_lock, reset ci_biglock_wanted. This
- * store must be unbuffered (immediately visible on the bus) in
- * order for non-interlocked mutex release to work correctly.
- * It must be visible before a mutex_exit() can execute on this
- * processor.
+ * store must be visible on other CPUs before a mutex_exit() on
+ * this CPU can test the has-waiters bit.
+ *
+ * This membar_enter matches the membar_enter in
+ * mutex_vector_enter. (Yes, not membar_exit -- the legacy
+ * naming is confusing, but store-before-load usually pairs
+ * with store-before-load, in the extremely rare cases where it
+ * is used at all.)
*
- * Note: only where CAS is available in hardware will this be
- * an unbuffered write, but non-interlocked release cannot be
- * done on CPUs without CAS in hardware.
+ * That way, mutex_vector_enter can't see this store
+ * ci->ci_biglock_wanted := owant until it has set the
+ * has-waiters bit.
*/
(void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant);
-
- /*
- * Issue a memory barrier as we have acquired a lock. This also
- * prevents stores from a following mutex_exit() being reordered
- * to occur before our store to ci_biglock_wanted above.
- */
#ifndef __HAVE_ATOMIC_AS_MEMBAR
membar_enter();
#endif
Index: src/sys/kern/kern_mutex.c
diff -u src/sys/kern/kern_mutex.c:1.102 src/sys/kern/kern_mutex.c:1.103
--- src/sys/kern/kern_mutex.c:1.102 Fri Jan 27 09:28:41 2023
+++ src/sys/kern/kern_mutex.c Thu Feb 23 14:57:29 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_mutex.c,v 1.102 2023/01/27 09:28:41 ozaki-r Exp $ */
+/* $NetBSD: kern_mutex.c,v 1.103 2023/02/23 14:57:29 riastradh Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2019 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
#define __MUTEX_PRIVATE
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.102 2023/01/27 09:28:41 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.103 2023/02/23 14:57:29 riastradh Exp $");
#include <sys/param.h>
#include <sys/atomic.h>
@@ -427,7 +427,7 @@ mutex_oncpu(uintptr_t owner)
if (ci && ci->ci_curlwp == l) {
/* Target is running; do we need to block? */
- return (ci->ci_biglock_wanted != l);
+ return (atomic_load_relaxed(&ci->ci_biglock_wanted) != l);
}
/* Not running. It may be safe to block now. */
