Module Name: src
Committed By: maxv
Date: Sun Apr 26 14:49:18 UTC 2020
Modified Files:
src/common/lib/libc/arch/i386/atomic: atomic.S
src/common/lib/libc/arch/x86_64/atomic: atomic.S
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/i386/include: frameasm.h
src/sys/arch/x86/x86: patch.c
Log Message:
Use the hotpatch framework for LFENCE/MFENCE.
To generate a diff of this commit:
cvs rdiff -u -r1.27 -r1.28 src/common/lib/libc/arch/i386/atomic/atomic.S
cvs rdiff -u -r1.21 -r1.22 src/common/lib/libc/arch/x86_64/atomic/atomic.S
cvs rdiff -u -r1.48 -r1.49 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.30 -r1.31 src/sys/arch/i386/include/frameasm.h
cvs rdiff -u -r1.41 -r1.42 src/sys/arch/x86/x86/patch.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/common/lib/libc/arch/i386/atomic/atomic.S
diff -u src/common/lib/libc/arch/i386/atomic/atomic.S:1.27 src/common/lib/libc/arch/i386/atomic/atomic.S:1.28
--- src/common/lib/libc/arch/i386/atomic/atomic.S:1.27 Sun Apr 26 13:59:44 2020
+++ src/common/lib/libc/arch/i386/atomic/atomic.S Sun Apr 26 14:49:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.27 2020/04/26 13:59:44 maxv Exp $ */
+/* $NetBSD: atomic.S,v 1.28 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -45,11 +45,15 @@
#ifdef _HARDKERNEL
#include "opt_xen.h"
#include <machine/frameasm.h>
-#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
+#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
+#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 7);
+#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 7);
+#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
#else
-#define LOCK lock
-#define ENDLABEL(a) /* nothing */
+#define LOCK lock
+#define HOTPATCH_SSE2_LFENCE /* nothing */
+#define HOTPATCH_SSE2_MFENCE /* nothing */
+#define ENDLABEL(a) /* nothing */
#endif
.text
@@ -177,11 +181,12 @@ ENTRY(_atomic_cas_32_ni)
END(_atomic_cas_32_ni)
ENTRY(_membar_consumer)
+ HOTPATCH_SSE2_LFENCE
+ /* 7 bytes of instructions */
LOCK
addl $0, -4(%esp)
ret
END(_membar_consumer)
-ENDLABEL(membar_consumer_end)
ENTRY(_membar_producer)
/* A store is enough */
@@ -190,11 +195,12 @@ ENTRY(_membar_producer)
END(_membar_producer)
ENTRY(_membar_sync)
+ HOTPATCH_SSE2_MFENCE
+ /* 7 bytes of instructions */
LOCK
addl $0, -4(%esp)
ret
END(_membar_sync)
-ENDLABEL(membar_sync_end)
#if defined(__HAVE_ATOMIC64_OPS) || defined(_KERNEL)
#ifdef XENPV
@@ -257,20 +263,6 @@ END(_atomic_cas_cx8)
ENDLABEL(_atomic_cas_cx8_end)
#endif /* __HAVE_ATOMIC64_OPS || _KERNEL */
-#ifdef _HARDKERNEL
-ENTRY(sse2_lfence)
- lfence
- ret
-END(sse2_lfence)
-ENDLABEL(sse2_lfence_end)
-
-ENTRY(sse2_mfence)
- mfence
- ret
-END(sse2_mfence)
-ENDLABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */
-
ALIAS(atomic_add_32,_atomic_add_32)
ALIAS(atomic_add_int,_atomic_add_32)
ALIAS(atomic_add_long,_atomic_add_32)
@@ -402,3 +394,19 @@ STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_
STRONG_ALIAS(_membar_enter,_membar_consumer)
STRONG_ALIAS(_membar_exit,_membar_producer)
+
+#ifdef _HARDKERNEL
+ .section .rodata
+
+LABEL(sse2_lfence)
+ lfence
+ ret
+ nop; nop; nop;
+LABEL(sse2_lfence_end)
+
+LABEL(sse2_mfence)
+ mfence
+ ret
+ nop; nop; nop;
+LABEL(sse2_mfence_end)
+#endif /* _HARDKERNEL */
Index: src/common/lib/libc/arch/x86_64/atomic/atomic.S
diff -u src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.21 src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.22
--- src/common/lib/libc/arch/x86_64/atomic/atomic.S:1.21 Sun Apr 26 13:59:44 2020
+++ src/common/lib/libc/arch/x86_64/atomic/atomic.S Sun Apr 26 14:49:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: atomic.S,v 1.21 2020/04/26 13:59:44 maxv Exp $ */
+/* $NetBSD: atomic.S,v 1.22 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -40,11 +40,13 @@
#ifdef _HARDKERNEL
#include <machine/frameasm.h>
-#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
-#define ENDLABEL(a) _ALIGN_TEXT; LABEL(a)
+#define LOCK HOTPATCH(HP_NAME_NOLOCK, 1); lock
+#define HOTPATCH_SSE2_LFENCE HOTPATCH(HP_NAME_SSE2_LFENCE, 8);
+#define HOTPATCH_SSE2_MFENCE HOTPATCH(HP_NAME_SSE2_MFENCE, 8);
#else
-#define LOCK lock
-#define ENDLABEL(a) /* nothing */
+#define LOCK lock
+#define HOTPATCH_SSE2_LFENCE /* nothing */
+#define HOTPATCH_SSE2_MFENCE /* nothing */
#endif
.text
@@ -254,11 +256,12 @@ END(_atomic_cas_64_ni)
/* memory barriers */
ENTRY(_membar_consumer)
+ HOTPATCH_SSE2_LFENCE
+ /* 8 bytes of instructions */
LOCK
addq $0, -8(%rsp)
ret
END(_membar_consumer)
-ENDLABEL(membar_consumer_end)
ENTRY(_membar_producer)
/* A store is enough */
@@ -267,25 +270,12 @@ ENTRY(_membar_producer)
END(_membar_producer)
ENTRY(_membar_sync)
+ HOTPATCH_SSE2_MFENCE
+ /* 8 bytes of instructions */
LOCK
addq $0, -8(%rsp)
ret
END(_membar_sync)
-ENDLABEL(membar_sync_end)
-
-#ifdef _HARDKERNEL
-ENTRY(sse2_lfence)
- lfence
- ret
-END(sse2_lfence)
-ENDLABEL(sse2_lfence_end)
-
-ENTRY(sse2_mfence)
- mfence
- ret
-END(sse2_mfence)
-ENDLABEL(sse2_mfence_end)
-#endif /* _HARDKERNEL */
ALIAS(atomic_add_32,_atomic_add_32)
ALIAS(atomic_add_64,_atomic_add_64)
@@ -425,3 +415,19 @@ STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_
STRONG_ALIAS(_membar_enter,_membar_consumer)
STRONG_ALIAS(_membar_exit,_membar_producer)
+
+#ifdef _HARDKERNEL
+ .section .rodata
+
+LABEL(sse2_lfence)
+ lfence
+ ret
+ nop; nop; nop; nop;
+LABEL(sse2_lfence_end)
+
+LABEL(sse2_mfence)
+ mfence
+ ret
+ nop; nop; nop; nop;
+LABEL(sse2_mfence_end)
+#endif /* _HARDKERNEL */
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.48 src/sys/arch/amd64/include/frameasm.h:1.49
--- src/sys/arch/amd64/include/frameasm.h:1.48 Sat Apr 25 15:26:16 2020
+++ src/sys/arch/amd64/include/frameasm.h Sun Apr 26 14:49:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.48 2020/04/25 15:26:16 bouyer Exp $ */
+/* $NetBSD: frameasm.h,v 1.49 2020/04/26 14:49:17 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -63,6 +63,8 @@
#define HP_NAME_SVS_ENTER_NMI 11
#define HP_NAME_SVS_LEAVE_NMI 12
#define HP_NAME_MDS_LEAVE 13
+#define HP_NAME_SSE2_LFENCE 14
+#define HP_NAME_SSE2_MFENCE 15
#define HOTPATCH(name, size) \
123: ; \
Index: src/sys/arch/i386/include/frameasm.h
diff -u src/sys/arch/i386/include/frameasm.h:1.30 src/sys/arch/i386/include/frameasm.h:1.31
--- src/sys/arch/i386/include/frameasm.h:1.30 Sat Apr 25 15:26:17 2020
+++ src/sys/arch/i386/include/frameasm.h Sun Apr 26 14:49:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.30 2020/04/25 15:26:17 bouyer Exp $ */
+/* $NetBSD: frameasm.h,v 1.31 2020/04/26 14:49:17 maxv Exp $ */
#ifndef _I386_FRAMEASM_H_
#define _I386_FRAMEASM_H_
@@ -48,6 +48,8 @@
#define HP_NAME_STAC 2
#define HP_NAME_NOLOCK 3
#define HP_NAME_RETFENCE 4
+#define HP_NAME_SSE2_LFENCE 5
+#define HP_NAME_SSE2_MFENCE 6
#define HOTPATCH(name, size) \
123: ; \
Index: src/sys/arch/x86/x86/patch.c
diff -u src/sys/arch/x86/x86/patch.c:1.41 src/sys/arch/x86/x86/patch.c:1.42
--- src/sys/arch/x86/x86/patch.c:1.41 Sun Apr 26 13:37:14 2020
+++ src/sys/arch/x86/x86/patch.c Sun Apr 26 14:49:17 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $ */
+/* $NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.41 2020/04/26 13:37:14 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.42 2020/04/26 14:49:17 maxv Exp $");
#include "opt_lockdebug.h"
#ifdef i386
@@ -69,15 +69,6 @@ void i686_mutex_spin_exit(int);
void i686_mutex_spin_exit_end(void);
void i686_mutex_spin_exit_patch(void);
-void membar_consumer(void);
-void membar_consumer_end(void);
-void membar_sync(void);
-void membar_sync_end(void);
-void sse2_lfence(void);
-void sse2_lfence_end(void);
-void sse2_mfence(void);
-void sse2_mfence_end(void);
-
void _atomic_cas_64(void);
void _atomic_cas_64_end(void);
void _atomic_cas_cx8(void);
@@ -219,16 +210,18 @@ x86_patch(bool early)
* ordinary non-temporal stores are always issued in
* program order to main memory and to other CPUs.
*/
- patchfunc(
- sse2_lfence, sse2_lfence_end,
- membar_consumer, membar_consumer_end,
- NULL
- );
- patchfunc(
- sse2_mfence, sse2_mfence_end,
- membar_sync, membar_sync_end,
- NULL
- );
+ extern uint8_t sse2_lfence, sse2_lfence_end;
+ extern uint8_t sse2_mfence, sse2_mfence_end;
+ uint8_t *bytes;
+ size_t size;
+
+ bytes = &sse2_lfence;
+ size = (size_t)&sse2_lfence_end - (size_t)&sse2_lfence;
+ x86_hotpatch(HP_NAME_SSE2_LFENCE, bytes, size);
+
+ bytes = &sse2_mfence;
+ size = (size_t)&sse2_mfence_end - (size_t)&sse2_mfence;
+ x86_hotpatch(HP_NAME_SSE2_MFENCE, bytes, size);
}
#ifdef i386
