Module Name: src
Committed By: thorpej
Date: Sun Jul 11 01:58:41 UTC 2021
Modified Files:
src/sys/arch/alpha/alpha: genassym.cf lock_stubs.s machdep.c
src/sys/arch/alpha/include: rwlock.h
Log Message:
Optimized fast-paths for rw_enter() / rw_tryenter() / rw_exit().
To generate a diff of this commit:
cvs rdiff -u -r1.28 -r1.29 src/sys/arch/alpha/alpha/genassym.cf
cvs rdiff -u -r1.4 -r1.5 src/sys/arch/alpha/alpha/lock_stubs.s
cvs rdiff -u -r1.373 -r1.374 src/sys/arch/alpha/alpha/machdep.c
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/alpha/include/rwlock.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/alpha/alpha/genassym.cf
diff -u src/sys/arch/alpha/alpha/genassym.cf:1.28 src/sys/arch/alpha/alpha/genassym.cf:1.29
--- src/sys/arch/alpha/alpha/genassym.cf:1.28 Sun Jul 11 01:54:42 2021
+++ src/sys/arch/alpha/alpha/genassym.cf Sun Jul 11 01:58:41 2021
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.28 2021/07/11 01:54:42 thorpej Exp $
+# $NetBSD: genassym.cf,v 1.29 2021/07/11 01:58:41 thorpej Exp $
#
# Copyright (c) 1982, 1990, 1993
@@ -66,12 +66,15 @@
# from: @(#)genassym.c 8.3 (Berkeley) 1/4/94
#
+quote #define __RWLOCK_PRIVATE
+
include <sys/param.h>
include <sys/buf.h>
include <sys/proc.h>
include <sys/sched.h>
include <sys/mbuf.h>
include <sys/msgbuf.h>
+include <sys/rwlock.h>
include <sys/syscall.h>
include <machine/cpu.h>
@@ -196,3 +199,9 @@ define CPU_INFO_IDLE_LWP offsetof(struct
define CPU_INFO_SSIR offsetof(struct cpu_info, ci_ssir)
define CPU_INFO_MTX_COUNT offsetof(struct cpu_info, ci_mtx_count)
define CPU_INFO_SIZEOF sizeof(struct cpu_info)
+
+# Bits in lock fields
+define RW_WRITE_WANTED RW_WRITE_WANTED
+define RW_WRITE_LOCKED RW_WRITE_LOCKED
+define RW_READ_INCR RW_READ_INCR
+define RW_READ_COUNT_SHIFT RW_READ_COUNT_SHIFT
Index: src/sys/arch/alpha/alpha/lock_stubs.s
diff -u src/sys/arch/alpha/alpha/lock_stubs.s:1.4 src/sys/arch/alpha/alpha/lock_stubs.s:1.5
--- src/sys/arch/alpha/alpha/lock_stubs.s:1.4 Fri Sep 4 02:54:56 2020
+++ src/sys/arch/alpha/alpha/lock_stubs.s Sun Jul 11 01:58:41 2021
@@ -1,11 +1,11 @@
-/* $NetBSD: lock_stubs.s,v 1.4 2020/09/04 02:54:56 thorpej Exp $ */
+/* $NetBSD: lock_stubs.s,v 1.5 2021/07/11 01:58:41 thorpej Exp $ */
/*-
- * Copyright (c) 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007, 2021 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
- * by Andrew Doran.
+ * by Andrew Doran, and by Jason R. Thorpe.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -34,7 +34,7 @@
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.s,v 1.4 2020/09/04 02:54:56 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.s,v 1.5 2021/07/11 01:58:41 thorpej Exp $");
#include "assym.h"
@@ -63,7 +63,7 @@ LEAF(_lock_cas, 3)
RET
3:
br 1b
-END(_lock_cas)
+ END(_lock_cas)
#if !defined(LOCKDEBUG)
@@ -72,7 +72,7 @@ END(_lock_cas)
*/
LEAF(mutex_enter, 1)
LDGP(pv)
- GET_CURLWP
+ GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
1:
mov v0, t1
ldq_l t2, 0(a0)
@@ -86,7 +86,7 @@ LEAF(mutex_enter, 1)
jmp (t12)
3:
br 1b
-END(mutex_enter)
+ END(mutex_enter)
/*
* void mutex_exit(kmutex_t *mtx);
@@ -94,7 +94,7 @@ END(mutex_enter)
LEAF(mutex_exit, 1)
LDGP(pv)
MB
- GET_CURLWP
+ GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
mov zero, t3
1:
ldq_l t2, 0(a0)
@@ -108,6 +108,162 @@ LEAF(mutex_exit, 1)
jmp (t12)
3:
br 1b
-END(mutex_exit)
+ END(mutex_exit)
+
+/*
+ * void rw_enter(krwlock_t *rwl, krw_t op);
+ *
+ * Acquire one hold on a RW lock.
+ */
+LEAF(rw_enter, 2)
+ LDGP(pv)
+
+ /*
+ * RW_READER == 0 (we have a compile-time assert in machdep.c
+ * to ensure this).
+ *
+ * Acquire for read is the most common case.
+ */
+ bne a1, 3f
+
+ /* Acquiring for read. */
+1: ldq_l t0, 0(a0)
+ and t0, (RW_WRITE_LOCKED|RW_WRITE_WANTED), t1
+ addq t0, RW_READ_INCR, t2
+ bne t1, 4f /* contended */
+ stq_c t2, 0(a0)
+ beq t2, 2f /* STQ_C failed; retry */
+ MB
+ RET
+
+2: br 1b
+
+3: /* Acquiring for write. */
+ GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
+ ldq_l t0, 0(a0)
+ or v0, RW_WRITE_LOCKED, t2
+ bne t0, 4f /* contended */
+ stq_c t2, 0(a0)
+ beq t2, 4f /* STQ_C failed; consider it contended */
+ MB
+ RET
+
+4: lda pv, rw_vector_enter
+ jmp (pv)
+ END(rw_enter)
+
+/*
+ * int rw_tryenter(krwlock_t *rwl, krw_t op);
+ *
+ * Try to acquire one hold on a RW lock.
+ */
+LEAF(rw_tryenter, 2)
+ LDGP(pv)
+
+ /* See above. */
+ bne a1, 3f
+
+ /* Acquiring for read. */
+1: ldq_l t0, 0(a0)
+ and t0, (RW_WRITE_LOCKED|RW_WRITE_WANTED), t1
+ addq t0, RW_READ_INCR, v0
+ bne t1, 4f /* contended */
+ stq_c v0, 0(a0)
+ beq v0, 2f /* STQ_C failed; retry */
+ MB
+ RET /* v0 contains non-zero LOCK_FLAG from STQ_C */
+
+2: br 1b
+
+ /* Acquiring for write. */
+3: GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
+ ldq_l t0, 0(a0)
+ or v0, RW_WRITE_LOCKED, v0
+ bne t0, 4f /* contended */
+ stq_c v0, 0(a0)
+ /*
+ * v0 now contains the LOCK_FLAG value from STQ_C, which is either
+ * 0 for failure, or non-zero for success. In either case, v0's
+ * value is correct. Go ahead and perform the memory barrier even
+ * in the failure case because we expect it to be rare and it saves
+ * a branch-not-taken instruction in the success case.
+ */
+ MB
+ RET
+
+4: mov zero, v0 /* return 0 (failure) */
+ RET
+ END(rw_tryenter)
+
+/*
+ * void rw_exit(krwlock_t *rwl);
+ *
+ * Release one hold on a RW lock.
+ */
+LEAF(rw_exit, 1)
+ LDGP(pv)
+ MB
+
+ /*
+ * Check for write-lock release, and get the owner/count field
+ * on its own for sanity-checking against expected values.
+ */
+ ldq a1, 0(a0)
+ and a1, RW_WRITE_LOCKED, t1
+ srl a1, RW_READ_COUNT_SHIFT, a2
+ bne t1, 3f
+
+ /*
+ * Releasing a read-lock. Make sure the count is non-zero.
+ * If it is zero, take the slow path where the juicy diagnostic
+ * checks are located.
+ */
+ beq a2, 4f
+
+ /*
+ * We do the following trick to check to see if we're releasing
+ * the last read-count and there are waiters:
+ *
+ * 1. Set v0 to 1.
+ * 2. Shift the new read count into t1.
+ * 3. Conditally move t1 to v0 based on low-bit-set of t0
+ * (RW_HAS_WAITERS). If RW_HAS_WAITERS is not set, then
+ * the move will not take place, and v0 will remain 1.
+ * Otherwise, v0 will contain the updated read count.
+ * 4. Jump to slow path if v0 == 0.
+ */
+1: ldq_l t0, 0(a0)
+ ldiq v0, 1
+ subq t0, RW_READ_INCR, t2
+ srl t2, RW_READ_COUNT_SHIFT, t1
+ cmovlbs t0, t1, v0
+ beq v0, 4f
+ stq_c t2, 0(a0)
+ beq t2, 2f /* STQ_C failed; try again */
+ RET
+
+2: br 1b
+
+ /*
+ * Releasing a write-lock. Make sure the owner field points
+ * to our LWP. If it does not, take the slow path where the
+ * juicy diagnostic checks are located. a2 contains the owner
+ * field shifted down. Shift it back up to compare to curlwp;
+ * this conveniently discards the bits we don't want to compare.
+ */
+3: GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
+ sll a2, RW_READ_COUNT_SHIFT, a2
+ mov zero, t2 /* fast-path write-unlock stores NULL */
+ cmpeq v0, a2, v0 /* v0 = (owner == curlwp) */
+ ldq_l t0, 0(a0)
+ beq v0, 4f /* owner field mismatch; need slow path */
+ blbs t0, 4f /* RW_HAS_WAITERS set; need slow-path */
+ stq_c t2, 0(a0)
+ beq t2, 4f /* STQ_C failed; need slow-path */
+ RET
+
+4: lda pv, rw_vector_exit
+ jmp (pv)
+ END(rw_exit)
#endif /* !LOCKDEBUG */
Index: src/sys/arch/alpha/alpha/machdep.c
diff -u src/sys/arch/alpha/alpha/machdep.c:1.373 src/sys/arch/alpha/alpha/machdep.c:1.374
--- src/sys/arch/alpha/alpha/machdep.c:1.373 Sun Jul 4 22:42:35 2021
+++ src/sys/arch/alpha/alpha/machdep.c Sun Jul 11 01:58:41 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: machdep.c,v 1.373 2021/07/04 22:42:35 thorpej Exp $ */
+/* $NetBSD: machdep.c,v 1.374 2021/07/11 01:58:41 thorpej Exp $ */
/*-
* Copyright (c) 1998, 1999, 2000, 2019, 2020 The NetBSD Foundation, Inc.
@@ -65,9 +65,11 @@
#include "opt_dec_3000_500.h"
#include "opt_execfmt.h"
+#define __RWLOCK_PRIVATE
+
#include <sys/cdefs.h> /* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.373 2021/07/04 22:42:35 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.374 2021/07/11 01:58:41 thorpej Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -95,6 +97,7 @@ __KERNEL_RCSID(0, "$NetBSD: machdep.c,v
#include <sys/kauth.h>
#include <sys/atomic.h>
#include <sys/cpu.h>
+#include <sys/rwlock.h>
#include <machine/kcore.h>
#include <machine/fpu.h>
@@ -133,6 +136,10 @@ int sigdebug = 0x0;
int sigpid = 0;
#endif
+/* Assert some assumptions made in lock_stubs.s */
+__CTASSERT(RW_READER == 0);
+__CTASSERT(RW_HAS_WAITERS == 1);
+
#include <machine/alpha.h>
#include "ksyms.h"
Index: src/sys/arch/alpha/include/rwlock.h
diff -u src/sys/arch/alpha/include/rwlock.h:1.5 src/sys/arch/alpha/include/rwlock.h:1.6
--- src/sys/arch/alpha/include/rwlock.h:1.5 Fri Nov 29 20:04:52 2019
+++ src/sys/arch/alpha/include/rwlock.h Sun Jul 11 01:58:41 2021
@@ -1 +1,37 @@
-/* $NetBSD: rwlock.h,v 1.5 2019/11/29 20:04:52 riastradh Exp $ */
+/* $NetBSD: rwlock.h,v 1.6 2021/07/11 01:58:41 thorpej Exp $ */
+
+/*-
+ * Copyright (c) 2002, 2006 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe and Andrew Doran.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _ALPHA_RWLOCK_H_
+#define _ALPHA_RWLOCK_H_
+
+#define __HAVE_RW_STUBS 1
+
+#endif /* _ALPHA_RWLOCK_H_ */
