Module Name: src
Committed By: thorpej
Date: Tue Jul 13 01:59:10 UTC 2021
Modified Files:
src/sys/arch/alpha/alpha: lock_stubs.s patch.c
Log Message:
As with membar_producer() and membar_sync(), initialize the fast-path
lock stubs with "unop" isns where memory barriers need to be, and if
we detect a multiprocessor system, patch those locations with "mb" insns.
To generate a diff of this commit:
cvs rdiff -u -r1.6 -r1.7 src/sys/arch/alpha/alpha/lock_stubs.s \
src/sys/arch/alpha/alpha/patch.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/alpha/alpha/lock_stubs.s
diff -u src/sys/arch/alpha/alpha/lock_stubs.s:1.6 src/sys/arch/alpha/alpha/lock_stubs.s:1.7
--- src/sys/arch/alpha/alpha/lock_stubs.s:1.6 Mon Jul 12 15:21:51 2021
+++ src/sys/arch/alpha/alpha/lock_stubs.s Tue Jul 13 01:59:10 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.s,v 1.6 2021/07/12 15:21:51 thorpej Exp $ */
+/* $NetBSD: lock_stubs.s,v 1.7 2021/07/13 01:59:10 thorpej Exp $ */
/*-
* Copyright (c) 2007, 2021 The NetBSD Foundation, Inc.
@@ -34,12 +34,16 @@
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.s,v 1.6 2021/07/12 15:21:51 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.s,v 1.7 2021/07/13 01:59:10 thorpej Exp $");
#include "assym.h"
#if defined(MULTIPROCESSOR)
-#define MB mb
+/*
+ * These 'unop' insns will be patched with 'mb' insns at run-time if
+ * the system has more than one processor.
+ */
+#define MB(label) label: unop
#else
#define MB /* nothing */
#endif
@@ -55,11 +59,11 @@ LEAF(_lock_cas, 3)
beq t1, 2f
stq_c v0, 0(a0)
beq v0, 3f
- MB
+ MB(.L__lock_cas_mb_1)
RET
2:
mov zero, v0
- MB
+ MB(.L__lock_cas_mb_2)
RET
3:
br 1b
@@ -79,7 +83,7 @@ LEAF(mutex_enter, 1)
bne t2, 2f
stq_c t1, 0(a0)
beq t1, 3f
- MB
+ MB(.L_mutex_enter_mb_1)
RET
2:
lda t12, mutex_vector_enter
@@ -93,7 +97,7 @@ LEAF(mutex_enter, 1)
*/
LEAF(mutex_exit, 1)
LDGP(pv)
- MB
+ MB(.L_mutex_exit_mb_1)
GET_CURLWP /* Note: GET_CURLWP clobbers v0, t0, t8...t11. */
mov zero, t3
1:
@@ -179,7 +183,7 @@ LEAF(mutex_spin_enter, 1);
bne t0, 2f /* contended */
stl_c t1, MUTEX_SIMPLELOCK(a1)
beq t1, 2f /* STL_C failed; consider contended */
- MB
+ MB(.L_mutex_spin_enter_mb_1)
RET
2:
mov a1, a0 /* restore first argument */
@@ -192,7 +196,7 @@ LEAF(mutex_spin_enter, 1);
*/
LEAF(mutex_spin_exit, 1)
LDGP(pv);
- MB
+ MB(.L_mutex_spin_exit_mb_1)
/*
* STEP 1: __cpu_simple_unlock(&mtx->mtx_lock)
@@ -247,7 +251,7 @@ LEAF(rw_enter, 2)
bne t1, 4f /* contended */
stq_c t2, 0(a0)
beq t2, 2f /* STQ_C failed; retry */
- MB
+ MB(.L_rw_enter_mb_1)
RET
2: br 1b
@@ -259,7 +263,7 @@ LEAF(rw_enter, 2)
bne t0, 4f /* contended */
stq_c t2, 0(a0)
beq t2, 4f /* STQ_C failed; consider it contended */
- MB
+ MB(.L_rw_enter_mb_2)
RET
4: lda pv, rw_vector_enter
@@ -284,7 +288,7 @@ LEAF(rw_tryenter, 2)
bne t1, 4f /* contended */
stq_c v0, 0(a0)
beq v0, 2f /* STQ_C failed; retry */
- MB
+ MB(.L_rw_tryenter_mb_1)
RET /* v0 contains non-zero LOCK_FLAG from STQ_C */
2: br 1b
@@ -302,7 +306,7 @@ LEAF(rw_tryenter, 2)
* in the failure case because we expect it to be rare and it saves
* a branch-not-taken instruction in the success case.
*/
- MB
+ MB(.L_rw_tryenter_mb_2)
RET
4: mov zero, v0 /* return 0 (failure) */
@@ -316,7 +320,7 @@ LEAF(rw_tryenter, 2)
*/
LEAF(rw_exit, 1)
LDGP(pv)
- MB
+ MB(.L_rw_exit_mb_1)
/*
* Check for write-lock release, and get the owner/count field
@@ -381,3 +385,27 @@ LEAF(rw_exit, 1)
END(rw_exit)
#endif /* !LOCKDEBUG */
+
+#if defined(MULTIPROCESSOR)
+/*
+ * Table of locations to patch with MB instructions on multiprocessor
+ * systems.
+ */
+ .section ".rodata"
+ .globl lock_stub_patch_table
+lock_stub_patch_table:
+ .quad .L__lock_cas_mb_1
+ .quad .L__lock_cas_mb_2
+#if !defined(LOCKDEBUG)
+ .quad .L_mutex_enter_mb_1
+ .quad .L_mutex_exit_mb_1
+ .quad .L_mutex_spin_enter_mb_1
+ .quad .L_mutex_spin_exit_mb_1
+ .quad .L_rw_enter_mb_1
+ .quad .L_rw_enter_mb_2
+ .quad .L_rw_tryenter_mb_1
+ .quad .L_rw_tryenter_mb_2
+ .quad .L_rw_exit_mb_1
+#endif /* ! LOCKDEBUG */
+ .quad 0 /* NULL terminator */
+#endif /* MULTIPROCESSOR */
Index: src/sys/arch/alpha/alpha/patch.c
diff -u src/sys/arch/alpha/alpha/patch.c:1.6 src/sys/arch/alpha/alpha/patch.c:1.7
--- src/sys/arch/alpha/alpha/patch.c:1.6 Wed Jul 7 03:30:35 2021
+++ src/sys/arch/alpha/alpha/patch.c Tue Jul 13 01:59:10 2021
@@ -1,7 +1,7 @@
-/* $NetBSD: patch.c,v 1.6 2021/07/07 03:30:35 thorpej Exp $ */
+/* $NetBSD: patch.c,v 1.7 2021/07/13 01:59:10 thorpej Exp $ */
/*-
- * Copyright (c) 2007 The NetBSD Foundation, Inc.
+ * Copyright (c) 2007, 2021 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.6 2021/07/07 03:30:35 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.7 2021/07/13 01:59:10 thorpej Exp $");
#include "opt_multiprocessor.h"
@@ -47,6 +47,8 @@ __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.
#include <machine/alpha.h>
#include <machine/intr.h>
+#include <alpha/alpha/db_instruction.h>
+
void _membar_producer(void);
void _membar_producer_end(void);
void _membar_producer_mp(void);
@@ -60,7 +62,32 @@ void _membar_sync_mp_end(void);
extern char alpha_copystr_bwx[], alpha_copystr_bwx_end[];
extern char alpha_copystr[], alpha_copystr_end[];
-static void __attribute__((__unused__))
+#if defined(MULTIPROCESSOR)
+extern uint32_t *lock_stub_patch_table[];
+
+static void
+patch_lock_stubs(void)
+{
+ alpha_instruction insn = {
+ .mem_format = {
+ .opcode = op_special,
+ .displacement = op_mb,
+ },
+ };
+ int i, s;
+
+ s = splhigh();
+
+ for (i = 0; lock_stub_patch_table[i] != NULL; i++) {
+ *lock_stub_patch_table[i] = insn.bits;
+ }
+ alpha_pal_imb();
+
+ splx(s);
+}
+#endif /* MULTIPROCESSOR */
+
+static void
patchfunc(void *from_s, void *from_e, void *to_s, void *to_e)
{
int s;
@@ -98,6 +125,8 @@ alpha_patch(bool is_mp)
KASSERT(curcpu()->ci_flags & CPUF_PRIMARY);
KASSERT((cpus_running & ~(1UL << cpu_number())) == 0);
+ patch_lock_stubs();
+
patchfunc(_membar_producer_mp, _membar_producer_mp_end,
_membar_producer, _membar_producer_end);
patchfunc(_membar_sync_mp, _membar_sync_mp_end,
