Module Name: src
Committed By: ad
Date: Wed Jan 22 11:40:17 UTC 2020
Modified Files:
src/sys/arch/amd64/amd64 [ad-namecache]: genassym.cf lock_stubs.S
src/sys/arch/amd64/include [ad-namecache]: rwlock.h
src/sys/arch/i386/i386 [ad-namecache]: genassym.cf lock_stubs.S
src/sys/arch/x86/include [ad-namecache]: rwlock.h
src/sys/kern [ad-namecache]: kern_rwlock.c
src/sys/sys [ad-namecache]: lwp.h rwlock.h
Log Message:
Back out previous.
To generate a diff of this commit:
cvs rdiff -u -r1.80.2.2 -r1.80.2.3 src/sys/arch/amd64/amd64/genassym.cf
cvs rdiff -u -r1.35.2.1 -r1.35.2.2 src/sys/arch/amd64/amd64/lock_stubs.S
cvs rdiff -u -r1.2.144.1 -r1.2.144.2 src/sys/arch/amd64/include/rwlock.h
cvs rdiff -u -r1.117.2.2 -r1.117.2.3 src/sys/arch/i386/i386/genassym.cf
cvs rdiff -u -r1.32.2.1 -r1.32.2.2 src/sys/arch/i386/i386/lock_stubs.S
cvs rdiff -u -r1.6.2.1 -r1.6.2.2 src/sys/arch/x86/include/rwlock.h
cvs rdiff -u -r1.59.2.3 -r1.59.2.4 src/sys/kern/kern_rwlock.c
cvs rdiff -u -r1.192.2.2 -r1.192.2.3 src/sys/sys/lwp.h
cvs rdiff -u -r1.12.2.1 -r1.12.2.2 src/sys/sys/rwlock.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/genassym.cf
diff -u src/sys/arch/amd64/amd64/genassym.cf:1.80.2.2 src/sys/arch/amd64/amd64/genassym.cf:1.80.2.3
--- src/sys/arch/amd64/amd64/genassym.cf:1.80.2.2 Sun Jan 19 21:08:29 2020
+++ src/sys/arch/amd64/amd64/genassym.cf Wed Jan 22 11:40:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.80.2.2 2020/01/19 21:08:29 ad Exp $
+# $NetBSD: genassym.cf,v 1.80.2.3 2020/01/22 11:40:16 ad Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -78,6 +78,7 @@ include <sys/resourcevar.h>
include <sys/device.h>
include <sys/mbuf.h>
include <sys/mutex.h>
+include <sys/rwlock.h>
include <sys/cpu_data.h>
include <sys/evcnt.h>
include <sys/cpu.h>
@@ -345,6 +346,15 @@ define MTX_IPL offsetof(struct kmutex,
define MTX_LOCK offsetof(struct kmutex, u.s.mtxs_lock)
define MTX_OWNER offsetof(struct kmutex, u.mtxa_owner)
+define RW_OWNER offsetof(struct krwlock, rw_owner)
+define RW_WRITE_LOCKED RW_WRITE_LOCKED
+define RW_WRITE_WANTED RW_WRITE_WANTED
+define RW_READ_INCR RW_READ_INCR
+define RW_HAS_WAITERS RW_HAS_WAITERS
+define RW_THREAD RW_THREAD
+define RW_READER RW_READER
+define RW_WRITER RW_WRITER
+
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
Index: src/sys/arch/amd64/amd64/lock_stubs.S
diff -u src/sys/arch/amd64/amd64/lock_stubs.S:1.35.2.1 src/sys/arch/amd64/amd64/lock_stubs.S:1.35.2.2
--- src/sys/arch/amd64/amd64/lock_stubs.S:1.35.2.1 Sun Jan 19 21:08:29 2020
+++ src/sys/arch/amd64/amd64/lock_stubs.S Wed Jan 22 11:40:16 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.S,v 1.35.2.1 2020/01/19 21:08:29 ad Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.35.2.2 2020/01/22 11:40:16 ad Exp $ */
/*
* Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -185,6 +185,126 @@ ENTRY(mutex_spin_exit)
END(mutex_spin_exit)
+/*
+ * void rw_enter(krwlock_t *rwl, krw_t op);
+ *
+ * Acquire one hold on a RW lock.
+ */
+ENTRY(rw_enter)
+ cmpl $RW_READER, %esi
+ jne 2f
+
+ /*
+ * Reader: this is the most common case.
+ */
+ movq (%rdi), %rax
+0:
+ testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
+ jnz 3f
+ leaq RW_READ_INCR(%rax), %rdx
+ LOCK
+ cmpxchgq %rdx, (%rdi)
+ jnz 1f
+ RET
+1:
+ jmp 0b
+
+ /*
+ * Writer: if the compare-and-set fails, don't bother retrying.
+ */
+2: movq CPUVAR(CURLWP), %rcx
+ xorq %rax, %rax
+ orq $RW_WRITE_LOCKED, %rcx
+ LOCK
+ cmpxchgq %rcx, (%rdi)
+ jnz 3f
+ RET
+3:
+ jmp _C_LABEL(rw_vector_enter)
+END(rw_enter)
+
+/*
+ * void rw_exit(krwlock_t *rwl);
+ *
+ * Release one hold on a RW lock.
+ */
+ENTRY(rw_exit)
+ movq (%rdi), %rax
+ testb $RW_WRITE_LOCKED, %al
+ jnz 2f
+
+ /*
+ * Reader
+ */
+0: testb $RW_HAS_WAITERS, %al
+ jnz 3f
+ cmpq $RW_READ_INCR, %rax
+ jb 3f
+ leaq -RW_READ_INCR(%rax), %rdx
+ LOCK
+ cmpxchgq %rdx, (%rdi)
+ jnz 1f
+ ret
+1:
+ jmp 0b
+
+ /*
+ * Writer
+ */
+2: leaq -RW_WRITE_LOCKED(%rax), %rdx
+ subq CPUVAR(CURLWP), %rdx
+ jnz 3f
+ LOCK
+ cmpxchgq %rdx, (%rdi)
+ jnz 3f
+ ret
+
+3: jmp _C_LABEL(rw_vector_exit)
+END(rw_exit)
+
+/*
+ * int rw_tryenter(krwlock_t *rwl, krw_t op);
+ *
+ * Try to acquire one hold on a RW lock.
+ */
+ENTRY(rw_tryenter)
+ cmpl $RW_READER, %esi
+ jne 2f
+
+ /*
+ * Reader: this is the most common case.
+ */
+ movq (%rdi), %rax
+0:
+ testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
+ jnz 4f
+ leaq RW_READ_INCR(%rax), %rdx
+ LOCK
+ cmpxchgq %rdx, (%rdi)
+ jnz 1f
+ movl %edx, %eax /* nonzero */
+ RET
+1:
+ jmp 0b
+
+ /*
+ * Writer: if the compare-and-set fails, don't bother retrying.
+ */
+2: movq CPUVAR(CURLWP), %rcx
+ xorq %rax, %rax
+ orq $RW_WRITE_LOCKED, %rcx
+ LOCK
+ cmpxchgq %rcx, (%rdi)
+ movl $0, %eax
+ setz %al
+3:
+ RET
+ ret
+4:
+ xorl %eax, %eax
+ jmp 3b
+END(rw_tryenter)
+
#endif /* LOCKDEBUG */
/*
Index: src/sys/arch/amd64/include/rwlock.h
diff -u src/sys/arch/amd64/include/rwlock.h:1.2.144.1 src/sys/arch/amd64/include/rwlock.h:1.2.144.2
--- src/sys/arch/amd64/include/rwlock.h:1.2.144.1 Sun Jan 19 21:11:00 2020
+++ src/sys/arch/amd64/include/rwlock.h Wed Jan 22 11:40:16 2020
@@ -0,0 +1,3 @@
+/* $NetBSD: rwlock.h,v 1.2.144.2 2020/01/22 11:40:16 ad Exp $ */
+
+#include <x86/rwlock.h>
Index: src/sys/arch/i386/i386/genassym.cf
diff -u src/sys/arch/i386/i386/genassym.cf:1.117.2.2 src/sys/arch/i386/i386/genassym.cf:1.117.2.3
--- src/sys/arch/i386/i386/genassym.cf:1.117.2.2 Sun Jan 19 21:08:29 2020
+++ src/sys/arch/i386/i386/genassym.cf Wed Jan 22 11:40:16 2020
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.117.2.2 2020/01/19 21:08:29 ad Exp $
+# $NetBSD: genassym.cf,v 1.117.2.3 2020/01/22 11:40:16 ad Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -79,6 +79,7 @@ include <sys/resourcevar.h>
include <sys/device.h>
include <sys/mbuf.h>
include <sys/mutex.h>
+include <sys/rwlock.h>
include <sys/cpu.h>
include <netinet/in.h>
@@ -353,6 +354,15 @@ define MTX_IPL offsetof(struct kmutex,
define MTX_LOCK offsetof(struct kmutex, mtx_lock)
define MTX_OWNER offsetof(struct kmutex, u.mtxa_owner)
+define RW_OWNER offsetof(struct krwlock, rw_owner)
+define RW_WRITE_LOCKED RW_WRITE_LOCKED
+define RW_WRITE_WANTED RW_WRITE_WANTED
+define RW_READ_INCR RW_READ_INCR
+define RW_HAS_WAITERS RW_HAS_WAITERS
+define RW_THREAD RW_THREAD
+define RW_READER RW_READER
+define RW_WRITER RW_WRITER
+
define EV_COUNT offsetof(struct evcnt, ev_count)
define OPTERON_MSR_PASSCODE OPTERON_MSR_PASSCODE
Index: src/sys/arch/i386/i386/lock_stubs.S
diff -u src/sys/arch/i386/i386/lock_stubs.S:1.32.2.1 src/sys/arch/i386/i386/lock_stubs.S:1.32.2.2
--- src/sys/arch/i386/i386/lock_stubs.S:1.32.2.1 Sun Jan 19 21:08:29 2020
+++ src/sys/arch/i386/i386/lock_stubs.S Wed Jan 22 11:40:16 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.S,v 1.32.2.1 2020/01/19 21:08:29 ad Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.32.2.2 2020/01/22 11:40:16 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.32.2.1 2020/01/19 21:08:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.32.2.2 2020/01/22 11:40:16 ad Exp $");
#include "opt_lockdebug.h"
@@ -97,6 +97,132 @@ ENTRY(mutex_exit)
jmp _C_LABEL(mutex_vector_exit)
END(mutex_exit)
+/*
+ * void rw_enter(krwlock_t *rwl, krw_t op);
+ *
+ * Acquire one hold on a RW lock.
+ */
+ENTRY(rw_enter)
+ movl 4(%esp), %edx
+ cmpl $RW_READER, 8(%esp)
+ jne 2f
+
+ /*
+ * Reader
+ */
+ movl (%edx), %eax
+0:
+ testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
+ jnz 3f
+ leal RW_READ_INCR(%eax), %ecx
+ LOCK(2)
+ cmpxchgl %ecx, (%edx)
+ jnz 1f
+ RET(2)
+1:
+ jmp 0b
+
+ /*
+ * Writer
+ */
+2: xorl %eax, %eax
+ movl %fs:CPU_INFO_CURLWP(%eax), %ecx
+ orl $RW_WRITE_LOCKED, %ecx
+ LOCK(3)
+ cmpxchgl %ecx, (%edx)
+ jnz 3f
+ RET(3)
+3:
+ jmp _C_LABEL(rw_vector_enter)
+END(rw_enter)
+
+/*
+ * void rw_exit(krwlock_t *rwl);
+ *
+ * Release one hold on a RW lock.
+ */
+ENTRY(rw_exit)
+ movl 4(%esp), %edx
+ movl (%edx), %eax
+ testb $RW_WRITE_LOCKED, %al
+ jnz 2f
+
+ /*
+ * Reader
+ */
+0: testb $RW_HAS_WAITERS, %al
+ jnz 3f
+ cmpl $RW_READ_INCR, %eax
+ jb 3f
+ leal -RW_READ_INCR(%eax), %ecx
+ LOCK(4)
+ cmpxchgl %ecx, (%edx)
+ jnz 1f
+ ret
+1:
+ jmp 0b
+
+ /*
+ * Writer
+ */
+2: leal -RW_WRITE_LOCKED(%eax), %ecx
+ subl CPUVAR(CURLWP), %ecx
+ jnz 3f
+ LOCK(5)
+ cmpxchgl %ecx, (%edx)
+ jnz 3f
+ ret
+
+ /*
+ * Slow path.
+ */
+3: jmp _C_LABEL(rw_vector_exit)
+END(rw_exit)
+
+/*
+ * int rw_tryenter(krwlock_t *rwl, krw_t op);
+ *
+ * Try to acquire one hold on a RW lock.
+ */
+ENTRY(rw_tryenter)
+ movl 4(%esp), %edx
+ cmpl $RW_READER, 8(%esp)
+ jne 2f
+
+ /*
+ * Reader
+ */
+ movl (%edx), %eax
+0:
+ testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
+ jnz 4f
+ leal RW_READ_INCR(%eax), %ecx
+ LOCK(12)
+ cmpxchgl %ecx, (%edx)
+ jnz 1f
+ movl %edx, %eax /* nonzero */
+ RET(4)
+1:
+ jmp 0b
+
+ /*
+ * Writer
+ */
+2:
+ xorl %eax, %eax
+ movl %fs:CPU_INFO_CURLWP(%eax), %ecx
+ orl $RW_WRITE_LOCKED, %ecx
+ LOCK(13)
+ cmpxchgl %ecx, (%edx)
+ movl $0, %eax
+ setz %al
+3:
+ RET(5)
+4:
+ xorl %eax, %eax
+ jmp 3b
+END(rw_tryenter)
+
#ifndef XENPV
/*
Index: src/sys/arch/x86/include/rwlock.h
diff -u src/sys/arch/x86/include/rwlock.h:1.6.2.1 src/sys/arch/x86/include/rwlock.h:1.6.2.2
--- src/sys/arch/x86/include/rwlock.h:1.6.2.1 Sun Jan 19 21:11:00 2020
+++ src/sys/arch/x86/include/rwlock.h Wed Jan 22 11:40:17 2020
@@ -0,0 +1,37 @@
+/* $NetBSD: rwlock.h,v 1.6.2.2 2020/01/22 11:40:17 ad Exp $ */
+
+/*-
+ * Copyright (c) 2002, 2006 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe and Andrew Doran.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _X86_RWLOCK_H_
+#define _X86_RWLOCK_H_
+
+#define __HAVE_RW_STUBS 1
+
+#endif /* _X86_RWLOCK_H_ */
Index: src/sys/kern/kern_rwlock.c
diff -u src/sys/kern/kern_rwlock.c:1.59.2.3 src/sys/kern/kern_rwlock.c:1.59.2.4
--- src/sys/kern/kern_rwlock.c:1.59.2.3 Sun Jan 19 21:08:29 2020
+++ src/sys/kern/kern_rwlock.c Wed Jan 22 11:40:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_rwlock.c,v 1.59.2.3 2020/01/19 21:08:29 ad Exp $ */
+/* $NetBSD: kern_rwlock.c,v 1.59.2.4 2020/01/22 11:40:17 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2009, 2019, 2020
@@ -36,23 +36,10 @@
*
* Solaris Internals: Core Kernel Architecture, Jim Mauro and
* Richard McDougall.
- *
- * The NetBSD implementation is different from that described in the book,
- * in that the locks are adaptive. Lock waiters spin wait while the lock
- * holders are on CPU (if the holds can be tracked: up to N per-thread).
- *
- * While spin waiting, threads compete for the lock without the assistance
- * of turnstiles. If a lock holder sleeps for any reason, the lock waiters
- * will also sleep in response and at that point turnstiles, priority
- * inheritance and strong efforts at ensuring fairness come into play.
- *
- * The adaptive behaviour is controlled by the RW_SPIN flag bit, which is
- * cleared by a lock owner that is going off the CPU, and set again by the
- * lock owner that releases the last hold on the lock.
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.59.2.3 2020/01/19 21:08:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.59.2.4 2020/01/22 11:40:17 ad Exp $");
#include "opt_lockdebug.h"
@@ -72,6 +59,8 @@ __KERNEL_RCSID(0, "$NetBSD: kern_rwlock.
#include <dev/lockstat.h>
+#include <machine/rwlock.h>
+
/*
* LOCKDEBUG
*/
@@ -115,6 +104,19 @@ do { \
#define RW_MEMBAR_PRODUCER() membar_producer()
#endif
+/*
+ * For platforms that do not provide stubs, or for the LOCKDEBUG case.
+ */
+#ifdef LOCKDEBUG
+#undef __HAVE_RW_STUBS
+#endif
+
+#ifndef __HAVE_RW_STUBS
+__strong_alias(rw_enter,rw_vector_enter);
+__strong_alias(rw_exit,rw_vector_exit);
+__strong_alias(rw_tryenter,rw_vector_tryenter);
+#endif
+
static void rw_abort(const char *, size_t, krwlock_t *, const char *);
static void rw_dump(const volatile void *, lockop_printer_t);
static lwp_t *rw_owner(wchan_t);
@@ -147,22 +149,6 @@ rw_cas(krwlock_t *rw, uintptr_t o, uintp
}
/*
- * rw_and:
- *
- * Do an atomic AND on the lock word.
- */
-static inline void
-rw_and(krwlock_t *rw, uintptr_t m)
-{
-
-#ifdef _LP64
- atomic_and_64(&rw->rw_owner, m);
-#else
- atomic_and_32(&rw->rw_owner, m);
-#endif
-}
-
-/*
* rw_swap:
*
* Do an atomic swap of the lock word. This is used only when it's
@@ -181,75 +167,6 @@ rw_swap(krwlock_t *rw, uintptr_t o, uint
}
/*
- * rw_hold_remember:
- *
- * Helper - when acquring a lock, record the new hold.
- */
-static inline uintptr_t
-rw_hold_remember(krwlock_t *rw, lwp_t *l)
-{
- int i;
-
- KASSERT(kpreempt_disabled());
-
- for (i = 0; i < __arraycount(l->l_rwlocks); i++) {
- if (__predict_true(l->l_rwlocks[i] == NULL)) {
- l->l_rwlocks[i] = rw;
- /*
- * Clear the write wanted flag on every acquire to
- * give readers a chance once again.
- */
- return ~RW_WRITE_WANTED;
- }
- }
-
- /*
- * Nowhere to track the hold so we lose: temporarily disable
- * spinning on the lock.
- */
- return ~(RW_WRITE_WANTED | RW_SPIN);
-}
-
-/*
- * rw_hold_forget:
- *
- * Helper - when releasing a lock, stop tracking the hold.
- */
-static inline void
-rw_hold_forget(krwlock_t *rw, lwp_t *l)
-{
- int i;
-
- KASSERT(kpreempt_disabled());
-
- for (i = 0; i < __arraycount(l->l_rwlocks); i++) {
- if (__predict_true(l->l_rwlocks[i] == rw)) {
- l->l_rwlocks[i] = NULL;
- return;
- }
- }
-}
-
-/*
- * rw_switch:
- *
- * Called by mi_switch() to indicate that an LWP is going off the CPU.
- */
-void
-rw_switch(void)
-{
- lwp_t *l = curlwp;
- int i;
-
- for (i = 0; i < __arraycount(l->l_rwlocks); i++) {
- if (l->l_rwlocks[i] != NULL) {
- rw_and(l->l_rwlocks[i], ~RW_SPIN);
- /* Leave in place for exit to clear. */
- }
- }
-}
-
-/*
* rw_dump:
*
* Dump the contents of a rwlock structure.
@@ -289,10 +206,15 @@ void
_rw_init(krwlock_t *rw, uintptr_t return_address)
{
+#ifdef LOCKDEBUG
+ /* XXX only because the assembly stubs can't handle RW_NODEBUG */
if (LOCKDEBUG_ALLOC(rw, &rwlock_lockops, return_address))
- rw->rw_owner = RW_SPIN;
+ rw->rw_owner = 0;
else
- rw->rw_owner = RW_SPIN | RW_NODEBUG;
+ rw->rw_owner = RW_NODEBUG;
+#else
+ rw->rw_owner = 0;
+#endif
}
void
@@ -311,18 +233,53 @@ void
rw_destroy(krwlock_t *rw)
{
- RW_ASSERT(rw, (rw->rw_owner & ~(RW_NODEBUG | RW_SPIN)) == 0);
+ RW_ASSERT(rw, (rw->rw_owner & ~RW_NODEBUG) == 0);
LOCKDEBUG_FREE((rw->rw_owner & RW_NODEBUG) == 0, rw);
}
/*
+ * rw_oncpu:
+ *
+ * Return true if an rwlock owner is running on a CPU in the system.
+ * If the target is waiting on the kernel big lock, then we must
+ * release it. This is necessary to avoid deadlock.
+ */
+static bool
+rw_oncpu(uintptr_t owner)
+{
+#ifdef MULTIPROCESSOR
+ struct cpu_info *ci;
+ lwp_t *l;
+
+ KASSERT(kpreempt_disabled());
+
+ if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS)) != RW_WRITE_LOCKED) {
+ return false;
+ }
+
+ /*
+ * See lwp_dtor() why dereference of the LWP pointer is safe.
+ * We must have kernel preemption disabled for that.
+ */
+ l = (lwp_t *)(owner & RW_THREAD);
+ ci = l->l_cpu;
+
+ if (ci && ci->ci_curlwp == l) {
+ /* Target is running; do we need to block? */
+ return (ci->ci_biglock_wanted != l);
+ }
+#endif
+ /* Not running. It may be safe to block now. */
+ return false;
+}
+
+/*
* rw_vector_enter:
*
- * The slow path for acquiring a rwlock, that considers all conditions.
- * Marked __noinline to prevent the compiler pulling it into rw_enter().
+ * Acquire a rwlock.
*/
-static void __noinline
-rw_vector_enter(krwlock_t *rw, const krw_t op, uintptr_t mask, uintptr_t ra)
+void
+rw_vector_enter(krwlock_t *rw, const krw_t op)
{
uintptr_t owner, incr, need_wait, set_wait, curthread, next;
turnstile_t *ts;
@@ -339,7 +296,6 @@ rw_vector_enter(krwlock_t *rw, const krw
RW_ASSERT(rw, !cpu_intr_p());
RW_ASSERT(rw, curthread != 0);
- RW_ASSERT(rw, kpreempt_disabled());
RW_WANTLOCK(rw, op);
if (panicstr == NULL) {
@@ -371,13 +327,15 @@ rw_vector_enter(krwlock_t *rw, const krw
LOCKSTAT_ENTER(lsflag);
+ KPREEMPT_DISABLE(curlwp);
for (owner = rw->rw_owner;;) {
/*
* Read the lock owner field. If the need-to-wait
* indicator is clear, then try to acquire the lock.
*/
if ((owner & need_wait) == 0) {
- next = rw_cas(rw, owner, (owner + incr) & mask);
+ next = rw_cas(rw, owner, (owner + incr) &
+ ~RW_WRITE_WANTED);
if (__predict_true(next == owner)) {
/* Got it! */
RW_MEMBAR_ENTER();
@@ -395,36 +353,11 @@ rw_vector_enter(krwlock_t *rw, const krw
rw_abort(__func__, __LINE__, rw,
"locking against myself");
}
-
/*
- * If the lock owner is running on another CPU, and there
- * are no existing waiters, then spin. Notes:
- *
- * 1) If an LWP on this CPU (possibly curlwp, or an LWP that
- * curlwp has interupted) holds kernel_lock, we can't spin
- * without a deadlock. The CPU that holds the rwlock may be
- * blocked trying to acquire kernel_lock, or there may be an
- * unseen chain of dependant locks. To defeat the potential
- * deadlock, this LWP needs to sleep (and thereby directly
- * drop the kernel_lock, or permit the interrupted LWP that
- * holds kernel_lock to complete its work).
- *
- * 2) If trying to acquire a write lock, and the lock is
- * currently read held, after a brief wait set the write
- * wanted bit to block out new readers and try to avoid
- * starvation. When the hold is acquired, we'll clear the
- * WRITE_WANTED flag to give readers a chance again. With
- * luck this should nudge things in the direction of
- * interleaving readers and writers when there is high
- * contention.
- *
- * 3) The spin wait can't be done in soft interrupt context,
- * because a lock holder could be pinned down underneath the
- * soft interrupt LWP (i.e. curlwp) on the same CPU. For
- * the lock holder to make progress and release the lock,
- * the soft interrupt needs to sleep.
+ * If the lock owner is running on another CPU, and
+ * there are no existing waiters, then spin.
*/
- if ((owner & RW_SPIN) != 0 && !cpu_softintr_p()) {
+ if (rw_oncpu(owner)) {
LOCKSTAT_START_TIMER(lsflag, spintime);
u_int count = SPINLOCK_BACKOFF_MIN;
do {
@@ -432,19 +365,7 @@ rw_vector_enter(krwlock_t *rw, const krw
SPINLOCK_BACKOFF(count);
KPREEMPT_DISABLE(curlwp);
owner = rw->rw_owner;
- if ((owner & need_wait) == 0)
- break;
- if (count != SPINLOCK_BACKOFF_MAX)
- continue;
- if (curcpu()->ci_biglock_count != 0)
- break;
- if (op == RW_WRITER &&
- (owner & RW_WRITE_LOCKED) == 0 &&
- (owner & RW_WRITE_WANTED) == 0) {
- (void)rw_cas(rw, owner,
- owner | RW_WRITE_WANTED);
- }
- } while ((owner & RW_SPIN) != 0);
+ } while (rw_oncpu(owner));
LOCKSTAT_STOP_TIMER(lsflag, spintime);
LOCKSTAT_COUNT(spincnt, 1);
if ((owner & need_wait) == 0)
@@ -458,18 +379,17 @@ rw_vector_enter(krwlock_t *rw, const krw
ts = turnstile_lookup(rw);
/*
- * Mark the rwlock as having waiters, and disable spinning.
- * If the set fails, then we may not need to sleep and
- * should spin again. Reload rw_owner now that we own
- * the turnstile chain lock.
+ * Mark the rwlock as having waiters. If the set fails,
+ * then we may not need to sleep and should spin again.
+ * Reload rw_owner because turnstile_lookup() may have
+ * spun on the turnstile chain lock.
*/
owner = rw->rw_owner;
- if ((owner & need_wait) == 0 ||
- ((owner & RW_SPIN) != 0 && !cpu_softintr_p())) {
+ if ((owner & need_wait) == 0 || rw_oncpu(owner)) {
turnstile_exit(rw);
continue;
}
- next = rw_cas(rw, owner, (owner | set_wait) & ~RW_SPIN);
+ next = rw_cas(rw, owner, owner | set_wait);
if (__predict_false(next != owner)) {
turnstile_exit(rw);
owner = next;
@@ -494,9 +414,11 @@ rw_vector_enter(krwlock_t *rw, const krw
LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK |
(op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime,
- (l->l_rwcallsite != 0 ? l->l_rwcallsite : ra));
+ (l->l_rwcallsite != 0 ? l->l_rwcallsite :
+ (uintptr_t)__builtin_return_address(0)));
LOCKSTAT_EVENT_RA(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime,
- (l->l_rwcallsite != 0 ? l->l_rwcallsite : ra));
+ (l->l_rwcallsite != 0 ? l->l_rwcallsite :
+ (uintptr_t)__builtin_return_address(0)));
LOCKSTAT_EXIT(lsflag);
RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
@@ -505,70 +427,11 @@ rw_vector_enter(krwlock_t *rw, const krw
}
/*
- * rw_enter:
- *
- * The fast path for acquiring a lock that considers only the
- * uncontended case. Falls back to rw_vector_enter().
- */
-void
-rw_enter(krwlock_t *rw, const krw_t op)
-{
- uintptr_t owner, incr, need_wait, curthread, next, mask;
- lwp_t *l;
-
- l = curlwp;
- curthread = (uintptr_t)l;
-
- RW_ASSERT(rw, !cpu_intr_p());
- RW_ASSERT(rw, curthread != 0);
- RW_WANTLOCK(rw, op);
-
- KPREEMPT_DISABLE(l);
- mask = rw_hold_remember(rw, l);
-
- /*
- * We play a slight trick here. If we're a reader, we want
- * increment the read count. If we're a writer, we want to
- * set the owner field and the WRITE_LOCKED bit.
- *
- * In the latter case, we expect those bits to be zero,
- * therefore we can use an add operation to set them, which
- * means an add operation for both cases.
- */
- if (__predict_true(op == RW_READER)) {
- incr = RW_READ_INCR;
- need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
- } else {
- RW_ASSERT(rw, op == RW_WRITER);
- incr = curthread | RW_WRITE_LOCKED;
- need_wait = RW_WRITE_LOCKED | RW_THREAD;
- }
-
- /*
- * Read the lock owner field. If the need-to-wait
- * indicator is clear, then try to acquire the lock.
- */
- owner = rw->rw_owner;
- if ((owner & need_wait) == 0) {
- next = rw_cas(rw, owner, (owner + incr) & mask);
- if (__predict_true(next == owner)) {
- /* Got it! */
- KPREEMPT_ENABLE(l);
- RW_MEMBAR_ENTER();
- return;
- }
- }
-
- rw_vector_enter(rw, op, mask, (uintptr_t)__builtin_return_address(0));
-}
-
-/*
* rw_vector_exit:
*
- * The slow path for releasing a rwlock, that considers all conditions.
- * Marked __noinline to prevent the compiler pulling it into rw_enter().
+ * Release a rwlock.
*/
-static void __noinline
+void
rw_vector_exit(krwlock_t *rw)
{
uintptr_t curthread, owner, decr, newown, next;
@@ -579,7 +442,6 @@ rw_vector_exit(krwlock_t *rw)
l = curlwp;
curthread = (uintptr_t)l;
RW_ASSERT(rw, curthread != 0);
- RW_ASSERT(rw, kpreempt_disabled());
/*
* Again, we use a trick. Since we used an add operation to
@@ -608,15 +470,9 @@ rw_vector_exit(krwlock_t *rw)
newown = (owner - decr);
if ((newown & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS)
break;
- /* Want spinning enabled if lock is becoming free. */
- if ((newown & RW_THREAD) == 0)
- newown |= RW_SPIN;
next = rw_cas(rw, owner, newown);
- if (__predict_true(next == owner)) {
- rw_hold_forget(rw, l);
- kpreempt_enable();
+ if (__predict_true(next == owner))
return;
- }
owner = next;
}
@@ -656,14 +512,12 @@ rw_vector_exit(krwlock_t *rw)
if (wcnt > 1)
newown |= RW_WRITE_WANTED;
rw_swap(rw, owner, newown);
- rw_hold_forget(rw, l);
turnstile_wakeup(ts, TS_WRITER_Q, 1, l);
} else {
/* Wake all writers and let them fight it out. */
newown = owner & RW_NODEBUG;
newown |= RW_WRITE_WANTED;
rw_swap(rw, owner, newown);
- rw_hold_forget(rw, l);
turnstile_wakeup(ts, TS_WRITER_Q, wcnt, NULL);
}
} else {
@@ -681,73 +535,19 @@ rw_vector_exit(krwlock_t *rw)
/* Wake up all sleeping readers. */
rw_swap(rw, owner, newown);
- rw_hold_forget(rw, l);
turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL);
}
- kpreempt_enable();
}
/*
- * rw_exit:
- *
- * The fast path for releasing a lock that considers only the
- * uncontended case. Falls back to rw_vector_exit().
- */
-void
-rw_exit(krwlock_t *rw)
-{
- uintptr_t curthread, owner, decr, newown, next;
- lwp_t *l;
-
- l = curlwp;
- curthread = (uintptr_t)l;
- RW_ASSERT(rw, curthread != 0);
-
- /*
- * Again, we use a trick. Since we used an add operation to
- * set the required lock bits, we can use a subtract to clear
- * them, which makes the read-release and write-release path
- * the same.
- */
- owner = rw->rw_owner;
- if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) {
- RW_UNLOCKED(rw, RW_WRITER);
- RW_ASSERT(rw, RW_OWNER(rw) == curthread);
- decr = curthread | RW_WRITE_LOCKED;
- } else {
- RW_UNLOCKED(rw, RW_READER);
- RW_ASSERT(rw, RW_COUNT(rw) != 0);
- decr = RW_READ_INCR;
- }
-
- /* Now try to release it. */
- RW_MEMBAR_EXIT();
- KPREEMPT_DISABLE(l);
- newown = (owner - decr);
- if (__predict_true((newown & (RW_THREAD | RW_HAS_WAITERS)) !=
- RW_HAS_WAITERS)) {
- /* Want spinning (re-)enabled if lock is becoming free. */
- if ((newown & RW_THREAD) == 0)
- newown |= RW_SPIN;
- next = rw_cas(rw, owner, newown);
- if (__predict_true(next == owner)) {
- rw_hold_forget(rw, l);
- KPREEMPT_ENABLE(l);
- return;
- }
- }
- rw_vector_exit(rw);
-}
-
-/*
- * rw_tryenter:
+ * rw_vector_tryenter:
*
* Try to acquire a rwlock.
*/
int
-rw_tryenter(krwlock_t *rw, const krw_t op)
+rw_vector_tryenter(krwlock_t *rw, const krw_t op)
{
- uintptr_t curthread, owner, incr, need_wait, next, mask;
+ uintptr_t curthread, owner, incr, need_wait, next;
lwp_t *l;
l = curlwp;
@@ -755,9 +555,6 @@ rw_tryenter(krwlock_t *rw, const krw_t o
RW_ASSERT(rw, curthread != 0);
- KPREEMPT_DISABLE(l);
- mask = rw_hold_remember(rw, l);
-
if (op == RW_READER) {
incr = RW_READ_INCR;
need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED;
@@ -768,12 +565,9 @@ rw_tryenter(krwlock_t *rw, const krw_t o
}
for (owner = rw->rw_owner;; owner = next) {
- if (__predict_false((owner & need_wait) != 0)) {
- rw_hold_forget(rw, l);
- KPREEMPT_ENABLE(l);
+ if (__predict_false((owner & need_wait) != 0))
return 0;
- }
- next = rw_cas(rw, owner, (owner + incr) & mask);
+ next = rw_cas(rw, owner, owner + incr);
if (__predict_true(next == owner)) {
/* Got it! */
break;
@@ -785,7 +579,6 @@ rw_tryenter(krwlock_t *rw, const krw_t o
RW_ASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) ||
(op == RW_READER && RW_COUNT(rw) != 0));
- KPREEMPT_ENABLE(l);
RW_MEMBAR_ENTER();
return 1;
}
@@ -823,7 +616,7 @@ rw_downgrade(krwlock_t *rw)
* waiters.
*/
if ((owner & RW_HAS_WAITERS) == 0) {
- newown = (owner & RW_NODEBUG) | RW_SPIN;
+ newown = (owner & RW_NODEBUG);
next = rw_cas(rw, owner, newown + RW_READ_INCR);
if (__predict_true(next == owner)) {
RW_LOCKED(rw, RW_READER);
@@ -850,15 +643,14 @@ rw_downgrade(krwlock_t *rw)
/*
* If there are no readers, just preserve the
* waiters bits, swap us down to one read hold and
- * return. Don't set the spin bit as nobody's
- * running yet.
+ * return.
*/
RW_ASSERT(rw, wcnt != 0);
RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0);
RW_ASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0);
newown = owner & RW_NODEBUG;
- newown = RW_READ_INCR | RW_HAS_WAITERS |
+ newown |= RW_READ_INCR | RW_HAS_WAITERS |
RW_WRITE_WANTED;
next = rw_cas(rw, owner, newown);
turnstile_exit(rw);
@@ -869,8 +661,7 @@ rw_downgrade(krwlock_t *rw)
* Give the lock to all blocked readers. We may
* retain one read hold if downgrading. If there is
* a writer waiting, new readers will be blocked
- * out. Don't set the spin bit as nobody's running
- * yet.
+ * out.
*/
newown = owner & RW_NODEBUG;
newown += (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR;
@@ -1005,14 +796,22 @@ rw_owner(wchan_t obj)
/*
* rw_owner_running:
*
- * Return true if a RW lock is unheld, or held and the owner is running
- * on a CPU. For the pagedaemon only - do not document or use in other
- * code.
+ * Return true if a RW lock is unheld, or write held and the owner is
+ * running on a CPU. For the pagedaemon.
*/
bool
rw_owner_running(const krwlock_t *rw)
{
- uintptr_t owner = rw->rw_owner;
+#ifdef MULTIPROCESSOR
+ uintptr_t owner;
+ bool rv;
- return (owner & RW_THREAD) == 0 || (owner & RW_SPIN) != 0;
+ kpreempt_disable();
+ owner = rw->rw_owner;
+ rv = (owner & RW_THREAD) == 0 || rw_oncpu(owner);
+ kpreempt_enable();
+ return rv;
+#else
+ return rw_owner(rw) == curlwp;
+#endif
}
Index: src/sys/sys/lwp.h
diff -u src/sys/sys/lwp.h:1.192.2.2 src/sys/sys/lwp.h:1.192.2.3
--- src/sys/sys/lwp.h:1.192.2.2 Sun Jan 19 21:08:30 2020
+++ src/sys/sys/lwp.h Wed Jan 22 11:40:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lwp.h,v 1.192.2.2 2020/01/19 21:08:30 ad Exp $ */
+/* $NetBSD: lwp.h,v 1.192.2.3 2020/01/22 11:40:17 ad Exp $ */
/*
* Copyright (c) 2001, 2006, 2007, 2008, 2009, 2010, 2019
@@ -186,7 +186,6 @@ struct lwp {
u_short l_exlocks; /* !: lockdebug: excl. locks held */
u_short l_psrefs; /* !: count of psref held */
u_short l_blcnt; /* !: count of kernel_lock held */
- struct krwlock *l_rwlocks[4]; /* !: tracks first N held rwlocks */
int l_nopreempt; /* !: don't preempt me! */
u_int l_dopreempt; /* s: kernel preemption pending */
int l_pflag; /* !: LWP private flags */
Index: src/sys/sys/rwlock.h
diff -u src/sys/sys/rwlock.h:1.12.2.1 src/sys/sys/rwlock.h:1.12.2.2
--- src/sys/sys/rwlock.h:1.12.2.1 Sun Jan 19 19:44:34 2020
+++ src/sys/sys/rwlock.h Wed Jan 22 11:40:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: rwlock.h,v 1.12.2.1 2020/01/19 19:44:34 ad Exp $ */
+/* $NetBSD: rwlock.h,v 1.12.2.2 2020/01/22 11:40:17 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc.
@@ -63,10 +63,10 @@ typedef struct krwlock krwlock_t;
* WRITE_LOCKED bit is clear, then the owner field is actually a count of
* the number of readers. The rw_owner field is laid out like so:
*
- * N 4 3 2 1 0
- * +---------------------------------------------------------+
- * | owner or read count | nodbug | wrlock | wrwant | wait |
- * +---------------------------------------------------------+
+ * N 5 4 3 2 1 0
+ * +------------------------------------------------------------------+
+ * | owner or read count | nodbug | <free> | wrlock | wrwant | wait |
+ * +------------------------------------------------------------------+
*/
#define RW_HAS_WAITERS 0x01UL /* lock has waiters */
#define RW_WRITE_WANTED 0x02UL /* >= 1 waiter is a writer */
@@ -84,6 +84,7 @@ void rw_vector_enter(krwlock_t *, const
void rw_vector_exit(krwlock_t *);
int rw_vector_tryenter(krwlock_t *, const krw_t);
void _rw_init(krwlock_t *, uintptr_t);
+bool rw_owner_running(const krwlock_t *);
#endif /* __RWLOCK_PRIVATE */
struct krwlock {
