Module Name: src
Committed By: maxv
Date: Sat May 18 13:32:12 UTC 2019
Modified Files:
src/sys/arch/amd64/amd64: amd64_trap.S locore.S
src/sys/arch/amd64/include: frameasm.h
Log Message:
Two changes in the CPU mitigations:
* Micro-optimize: put every mitigation in the same branch. This removes
two branches in each exc/int return path, and removes all branches in
the syscall return path.
* Modify the SpectreV2 mitigation to be compatible with SpectreV4. I
recently realized that both couldn't be enabled at the same time on
Intel. This is because initially, when there was just SpectreV2, we
could reset the whole IA32_SPEC_CTRL MSR. But then Intel added another
bit in it for SpectreV4, so it isn't right to reset it entirely
anymore. SSBD needs to stay.
To generate a diff of this commit:
cvs rdiff -u -r1.47 -r1.48 src/sys/arch/amd64/amd64/amd64_trap.S
cvs rdiff -u -r1.184 -r1.185 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.43 -r1.44 src/sys/arch/amd64/include/frameasm.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/amd64_trap.S
diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.47 src/sys/arch/amd64/amd64/amd64_trap.S:1.48
--- src/sys/arch/amd64/amd64/amd64_trap.S:1.47 Tue May 14 16:59:25 2019
+++ src/sys/arch/amd64/amd64/amd64_trap.S Sat May 18 13:32:12 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.47 2019/05/14 16:59:25 maxv Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.48 2019/05/18 13:32:12 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -253,7 +253,12 @@ IDTVEC(trap02)
ZTRAP_NJ(T_NMI)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
+
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1f
IBRS_ENTER
+1:
+
cld
SMAP_ENABLE
movw %gs,TF_GS(%rsp)
@@ -281,9 +286,13 @@ IDTVEC(trap02)
call _C_LABEL(nmitrap)
.Lnmileave:
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1f
MDS_LEAVE
- SVS_LEAVE_NMI
IBRS_LEAVE
+1:
+
+ SVS_LEAVE_NMI
INTR_RESTORE_GPRS
addq $TF_REGSIZE+16,%rsp
iretq
@@ -353,12 +362,15 @@ IDTVEC(trap08)
TRAP_NJ(T_DOUBLEFLT)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
- IBRS_ENTER
- SVS_ENTER_ALTSTACK
+
testb $SEL_UPL,TF_CS(%rsp)
jz 1f
+ IBRS_ENTER
swapgs
1:
+
+ SVS_ENTER_ALTSTACK
+
cld
SMAP_ENABLE
movw %gs,TF_GS(%rsp)
@@ -370,15 +382,15 @@ IDTVEC(trap08)
incq CPUVAR(NTRAP)
call _C_LABEL(doubletrap)
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1f
MDS_LEAVE
SVS_LEAVE_ALTSTACK
IBRS_LEAVE
- INTR_RESTORE_GPRS
-
- testb $SEL_UPL,TF_CS(%rsp)
- jz 1f
swapgs
1:
+
+ INTR_RESTORE_GPRS
addq $TF_REGSIZE+16,%rsp
iretq
#endif
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.184 src/sys/arch/amd64/amd64/locore.S:1.185
--- src/sys/arch/amd64/amd64/locore.S:1.184 Sat May 18 07:58:58 2019
+++ src/sys/arch/amd64/amd64/locore.S Sat May 18 13:32:12 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.184 2019/05/18 07:58:58 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.185 2019/05/18 13:32:12 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1574,25 +1574,26 @@ END(pagezero)
.type intrfastexit,@function
LABEL(intrfastexit)
NOT_XEN(cli;)
+
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz .Lkexit
+
MDS_LEAVE
SVS_LEAVE
IBRS_LEAVE
INTR_RESTORE_GPRS
addq $(TF_REGSIZE+16),%rsp /* iret frame */
+ SWAPGS
- testb $SEL_UPL,TF_BACKW(TF_CS, %rsp)
- jz .Lkexit
cmpw $LSEL(LUCODE_SEL, SEL_UPL),TF_BACKW(TF_CS, %rsp)
- je .Luexit64
+ je do_iret
cmpw $GSEL(GUCODE_SEL, SEL_UPL),TF_BACKW(TF_CS, %rsp)
- je .Luexit64
+ je do_iret
#ifdef XENPV
cmpw $FLAT_RING3_CS64,TF_BACKW(TF_CS, %rsp)
- je .Luexit64
+ je do_iret
#endif
-.Luexit32:
- SWAPGS
do_mov_es:
movw TF_BACKW(TF_ES, %rsp),%es
do_mov_ds:
@@ -1603,13 +1604,13 @@ do_mov_fs:
do_mov_gs:
movw TF_BACKW(TF_GS, %rsp),%gs
#endif
- jmp .Lkexit
-.Luexit64:
- SWAPGS
+do_iret:
+ iretq
.Lkexit:
-do_iret:
+ INTR_RESTORE_GPRS
+ addq $(TF_REGSIZE+16),%rsp /* iret frame */
iretq
END(intrfastexit)
@@ -1643,12 +1644,9 @@ LABEL(svs_enter_nmi)
LABEL(svs_enter_nmi_end)
LABEL(svs_leave)
- testb $SEL_UPL,TF_CS(%rsp)
- jz 1234f
movq CPUVAR(URSP0),%rsp
movq CPUVAR(UPDIRPA),%rax
movq %rax,%cr3
-1234:
LABEL(svs_leave_end)
LABEL(svs_leave_altstack)
@@ -1671,20 +1669,17 @@ LABEL(svs_leave_nmi_end)
/* IBRS <- 1 */
LABEL(ibrs_enter)
movl $MSR_IA32_SPEC_CTRL,%ecx
- movl $IA32_SPEC_CTRL_IBRS,%eax
- movl $(IA32_SPEC_CTRL_IBRS >> 32),%edx
+ rdmsr
+ orl $IA32_SPEC_CTRL_IBRS,%eax
wrmsr
LABEL(ibrs_enter_end)
/* IBRS <- 0 */
LABEL(ibrs_leave)
- testb $SEL_UPL,TF_CS(%rsp)
- jz 1234f
movl $MSR_IA32_SPEC_CTRL,%ecx
- xorl %eax,%eax
- xorl %edx,%edx
+ rdmsr
+ andl $~IA32_SPEC_CTRL_IBRS,%eax
wrmsr
-1234:
LABEL(ibrs_leave_end)
LABEL(noibrs_enter)
@@ -1698,12 +1693,9 @@ LABEL(noibrs_leave_end)
.globl mds_leave, mds_leave_end
LABEL(mds_leave)
- testb $SEL_UPL,TF_CS(%rsp)
- jz 1234f
pushq $GSEL(GDATA_SEL, SEL_KPL)
verw (%rsp)
addq $8,%rsp
-1234:
LABEL(mds_leave_end)
LABEL(nomds_leave)
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.43 src/sys/arch/amd64/include/frameasm.h:1.44
--- src/sys/arch/amd64/include/frameasm.h:1.43 Tue May 14 16:59:25 2019
+++ src/sys/arch/amd64/include/frameasm.h Sat May 18 13:32:12 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.43 2019/05/14 16:59:25 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.44 2019/05/18 13:32:12 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -70,7 +70,7 @@
* IBRS
*/
-#define IBRS_ENTER_BYTES 17
+#define IBRS_ENTER_BYTES 12
#define IBRS_ENTER \
HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \
NOIBRS_ENTER
@@ -78,7 +78,7 @@
.byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \
.fill (IBRS_ENTER_BYTES-2),1,0xCC
-#define IBRS_LEAVE_BYTES 21
+#define IBRS_LEAVE_BYTES 12
#define IBRS_LEAVE \
HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \
NOIBRS_LEAVE
@@ -90,7 +90,7 @@
* MDS
*/
-#define MDS_LEAVE_BYTES 20
+#define MDS_LEAVE_BYTES 10
#define MDS_LEAVE \
HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \
NOMDS_LEAVE
@@ -156,7 +156,7 @@
HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \
NOSVS_ENTER
-#define SVS_LEAVE_BYTES 31
+#define SVS_LEAVE_BYTES 21
#define NOSVS_LEAVE \
.byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \
.fill (SVS_LEAVE_BYTES-2),1,0xCC
