Module Name: src
Committed By: maxv
Date: Fri May 1 09:40:48 UTC 2020
Modified Files:
src/sys/arch/i386/i386: lock_stubs.S spl.S
src/sys/arch/i386/include: frameasm.h
src/sys/arch/x86/x86: patch.c
Log Message:
Switch the rest of i386 to the x86_hotpatch mechanism.
To generate a diff of this commit:
cvs rdiff -u -r1.34 -r1.35 src/sys/arch/i386/i386/lock_stubs.S
cvs rdiff -u -r1.53 -r1.54 src/sys/arch/i386/i386/spl.S
cvs rdiff -u -r1.32 -r1.33 src/sys/arch/i386/include/frameasm.h
cvs rdiff -u -r1.45 -r1.46 src/sys/arch/x86/x86/patch.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/i386/i386/lock_stubs.S
diff -u src/sys/arch/i386/i386/lock_stubs.S:1.34 src/sys/arch/i386/i386/lock_stubs.S:1.35
--- src/sys/arch/i386/i386/lock_stubs.S:1.34 Fri May 1 09:17:58 2020
+++ src/sys/arch/i386/i386/lock_stubs.S Fri May 1 09:40:47 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: lock_stubs.S,v 1.34 2020/05/01 09:17:58 maxv Exp $ */
+/* $NetBSD: lock_stubs.S,v 1.35 2020/05/01 09:40:47 maxv Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -35,7 +35,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.34 2020/05/01 09:17:58 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.35 2020/05/01 09:40:47 maxv Exp $");
#include "opt_lockdebug.h"
@@ -255,9 +255,10 @@ END(mutex_spin_enter)
#ifndef XENPV
/*
- * Release a spin mutex and post a store fence.
+ * Release a spin mutex and post a store fence. Must occupy 96 bytes.
*/
ENTRY(mutex_spin_exit)
+ HOTPATCH(HP_NAME_MUTEX_EXIT, 96)
movl 4(%esp), %edx
movl CPUVAR(MTX_OLDSPL), %ecx
incl CPUVAR(MTX_COUNT)
@@ -273,7 +274,6 @@ ENTRY(mutex_spin_exit)
1: ret
.space 32, 0xCC
.align 32
-LABEL(mutex_spin_exit_end)
END(mutex_spin_exit)
#else /* XENPV */
STRONG_ALIAS(mutex_spin_exit, i686_mutex_spin_exit)
@@ -281,7 +281,7 @@ STRONG_ALIAS(mutex_spin_exit, i686_mutex
/*
* Patch for i686 CPUs where cli/sti is prohibitively expensive.
- * Must be the same size as mutex_spin_exit().
+ * Must be the same size as mutex_spin_exit(), that is, 96 bytes.
*/
ENTRY(i686_mutex_spin_exit)
mov 4(%esp),%edx
Index: src/sys/arch/i386/i386/spl.S
diff -u src/sys/arch/i386/i386/spl.S:1.53 src/sys/arch/i386/i386/spl.S:1.54
--- src/sys/arch/i386/i386/spl.S:1.53 Fri May 1 09:23:43 2020
+++ src/sys/arch/i386/i386/spl.S Fri May 1 09:40:47 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: spl.S,v 1.53 2020/05/01 09:23:43 maxv Exp $ */
+/* $NetBSD: spl.S,v 1.54 2020/05/01 09:40:47 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.53 2020/05/01 09:23:43 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.54 2020/05/01 09:40:47 maxv Exp $");
#include "opt_ddb.h"
#include "opt_spldebug.h"
@@ -71,11 +71,12 @@ END(splraise)
/*
* void spllower(int s);
*
- * spllower() for i486 and Pentium. Must be the same size as
- * cx8_spllower(). This must use pushf/cli/popf as it is used
- * early in boot where interrupts are disabled via eflags/IE.
+ * spllower() for i486 and Pentium. Must be the same size as cx8_spllower(),
+ * that is, 64 bytes. This must use pushf/cli/popf as it is used early in boot
+ * where interrupts are disabled via eflags/IE.
*/
ENTRY(spllower)
+ HOTPATCH(HP_NAME_SPLLOWER, 64)
#ifdef SPLDEBUG
movl 4(%esp),%ecx
pushl %ebp
@@ -101,7 +102,6 @@ ENTRY(spllower)
popf
jmp _C_LABEL(Xspllower)
.align 32
-LABEL(spllower_end)
END(spllower)
#else /* XENPV */
STRONG_ALIAS(spllower, cx8_spllower)
@@ -110,10 +110,10 @@ STRONG_ALIAS(spllower, cx8_spllower)
/*
* void cx8_spllower(int s);
*
- * spllower() optimized for Pentium Pro and later, which have long
- * pipelines that will be stalled by pushf/cli/popf. Must be the
- * same size as spllower(). Does not need to restore eflags/IE as
- * is patched in once autoconf is underway.
+ * spllower() optimized for Pentium Pro and later, which have long pipelines
+ * that will be stalled by pushf/cli/popf. Must be the same size as
+ * spllower(), ie 64 bytes. Does not need to restore eflags/IE as is patched
+ * in once autoconf is underway.
*
* For cmpxchg8b, edx/ecx are the high words and eax/ebx the low.
*
Index: src/sys/arch/i386/include/frameasm.h
diff -u src/sys/arch/i386/include/frameasm.h:1.32 src/sys/arch/i386/include/frameasm.h:1.33
--- src/sys/arch/i386/include/frameasm.h:1.32 Fri May 1 08:32:50 2020
+++ src/sys/arch/i386/include/frameasm.h Fri May 1 09:40:47 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.32 2020/05/01 08:32:50 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.33 2020/05/01 09:40:47 maxv Exp $ */
#ifndef _I386_FRAMEASM_H_
#define _I386_FRAMEASM_H_
@@ -51,6 +51,8 @@
#define HP_NAME_SSE2_LFENCE 5
#define HP_NAME_SSE2_MFENCE 6
#define HP_NAME_CAS_64 7
+#define HP_NAME_SPLLOWER 8
+#define HP_NAME_MUTEX_EXIT 9
#define HOTPATCH(name, size) \
123: ; \
Index: src/sys/arch/x86/x86/patch.c
diff -u src/sys/arch/x86/x86/patch.c:1.45 src/sys/arch/x86/x86/patch.c:1.46
--- src/sys/arch/x86/x86/patch.c:1.45 Fri May 1 09:17:58 2020
+++ src/sys/arch/x86/x86/patch.c Fri May 1 09:40:47 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: patch.c,v 1.45 2020/05/01 09:17:58 maxv Exp $ */
+/* $NetBSD: patch.c,v 1.46 2020/05/01 09:40:47 maxv Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.45 2020/05/01 09:17:58 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.46 2020/05/01 09:40:47 maxv Exp $");
#include "opt_lockdebug.h"
#ifdef i386
@@ -58,26 +58,6 @@ struct hotpatch {
void *addr;
} __packed;
-void spllower(int);
-void spllower_end(void);
-void cx8_spllower(int);
-void cx8_spllower_end(void);
-
-void mutex_spin_exit_end(void);
-void i686_mutex_spin_exit(int);
-void i686_mutex_spin_exit_end(void);
-
-static void __unused
-patchfunc(void *from_s, void *from_e, void *to_s, void *to_e)
-{
-
- if ((uintptr_t)from_e - (uintptr_t)from_s !=
- (uintptr_t)to_e - (uintptr_t)to_s)
- panic("patchfunc: sizes do not match (from=%p)", from_s);
-
- memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
-}
-
static inline void __unused
patchbytes(void *addr, const uint8_t *bytes, size_t size)
{
@@ -207,16 +187,18 @@ x86_patch(bool early)
#if !defined(SPLDEBUG)
if (!early && (cpu_feature[0] & CPUID_CX8) != 0) {
/* Faster splx(), mutex_spin_exit(). */
- patchfunc(
- cx8_spllower, cx8_spllower_end,
- spllower, spllower_end
- );
+ extern uint8_t cx8_spllower, cx8_spllower_end;
+ extern uint8_t i686_mutex_spin_exit, i686_mutex_spin_exit_end;
+
+ bytes = &cx8_spllower;
+ size = (size_t)&cx8_spllower_end - (size_t)&cx8_spllower;
+ x86_hotpatch(HP_NAME_SPLLOWER, bytes, size);
+
#if !defined(LOCKDEBUG)
- patchfunc(
- i686_mutex_spin_exit, i686_mutex_spin_exit_end,
- mutex_spin_exit, mutex_spin_exit_end
- );
-#endif /* !LOCKDEBUG */
+ bytes = &i686_mutex_spin_exit;
+ size = (size_t)&i686_mutex_spin_exit_end - (size_t)&i686_mutex_spin_exit;
+ x86_hotpatch(HP_NAME_MUTEX_EXIT, bytes, size);
+#endif
}
#endif /* !SPLDEBUG */
#endif /* i386 */
