Module Name: src
Committed By: maxv
Date: Wed Mar 28 16:02:49 UTC 2018
Modified Files:
src/sys/arch/amd64/amd64: amd64_trap.S locore.S
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/x86/x86: spectre.c
Log Message:
Add the IBRS mitigation for SpectreV2 on amd64.
Different operations are performed during context transitions:
user->kernel: IBRS <- 1
kernel->user: IBRS <- 0
And during context switches:
user->user: IBPB <- 0
kernel->user: IBPB <- 0
[user->kernel:IBPB <- 0 this one may not be needed]
We use two macros, IBRS_ENTER and IBRS_LEAVE, to set the IBRS bit. The
thing is hotpatched for better performance, like SVS.
The idea is that IBRS is a "privileged" bit, which is set to 1 in kernel
mode and 0 in user mode. To protect the branch predictor between user
processes (which are of the same privilege), we use the IBPB barrier.
The Intel manual also talks about (MWAIT/HLT)+HyperThreading, and says
that when using either of the two instructions IBRS must be disabled for
better performance on the core. I'm not totally sure about this part, so
I'm not adding it now.
IBRS is available only when the Intel microcode update is applied. The
mitigation must be enabled manually with machdep.spectreV2.mitigated.
Tested by msaitoh a week ago (but I adapted a few things since). Probably
more changes to come.
To generate a diff of this commit:
cvs rdiff -u -r1.39 -r1.40 src/sys/arch/amd64/amd64/amd64_trap.S
cvs rdiff -u -r1.159 -r1.160 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.37 -r1.38 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/x86/x86/spectre.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/amd64_trap.S
diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.39 src/sys/arch/amd64/amd64/amd64_trap.S:1.40
--- src/sys/arch/amd64/amd64/amd64_trap.S:1.39 Tue Mar 20 18:27:58 2018
+++ src/sys/arch/amd64/amd64/amd64_trap.S Wed Mar 28 16:02:49 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.39 2018/03/20 18:27:58 maxv Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.40 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -134,6 +134,7 @@ IDTVEC(trap02)
ZTRAP_NJ(T_NMI)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
+ IBRS_ENTER
SVS_ENTER_ALTSTACK
cld
SMAP_ENABLE
@@ -162,6 +163,7 @@ IDTVEC(trap02)
SVS_LEAVE_ALTSTACK
.Lnmileave:
+ IBRS_LEAVE
INTR_RESTORE_GPRS
addq $TF_REGSIZE+16,%rsp
iretq
@@ -231,6 +233,7 @@ IDTVEC(trap08)
TRAP_NJ(T_DOUBLEFLT)
subq $TF_REGSIZE,%rsp
INTR_SAVE_GPRS
+ IBRS_ENTER
SVS_ENTER_ALTSTACK
testb $SEL_UPL,TF_CS(%rsp)
jz 1f
@@ -248,6 +251,7 @@ IDTVEC(trap08)
call _C_LABEL(doubletrap)
SVS_LEAVE_ALTSTACK
+ IBRS_LEAVE
INTR_RESTORE_GPRS
testb $SEL_UPL,TF_CS(%rsp)
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.159 src/sys/arch/amd64/amd64/locore.S:1.160
--- src/sys/arch/amd64/amd64/locore.S:1.159 Tue Mar 20 18:27:58 2018
+++ src/sys/arch/amd64/amd64/locore.S Wed Mar 28 16:02:49 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.159 2018/03/20 18:27:58 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.160 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1098,6 +1098,10 @@ ENTRY(cpu_switchto)
popq %rdx
#endif
+ pushq %rdx
+ callq _C_LABEL(speculation_barrier)
+ popq %rdx
+
/* Switch to newlwp's stack. */
movq L_PCB(%r12),%r14
movq PCB_RSP(%r14),%rsp
@@ -1413,6 +1417,7 @@ IDTVEC(\name)
cld
#endif
INTR_SAVE_GPRS
+ IBRS_ENTER
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_DS(%rsp)
movw $GSEL(GUDATA_SEL, SEL_UPL),TF_ES(%rsp)
movw $0,TF_FS(%rsp)
@@ -1464,6 +1469,7 @@ IDTVEC_END(osyscall)
_ALIGN_TEXT
LABEL(syscall_sysret)
SVS_LEAVE
+ IBRS_LEAVE
INTR_RESTORE_GPRS
SWAPGS
#ifndef XEN
@@ -1554,6 +1560,7 @@ END(pagezero)
LABEL(intrfastexit)
NOT_XEN(cli;)
SVS_LEAVE
+ IBRS_LEAVE
INTR_RESTORE_GPRS
addq $(TF_REGSIZE+16),%rsp /* iret frame */
@@ -1649,3 +1656,25 @@ LABEL(nosvs_leave_altstack)
NOSVS_LEAVE_ALTSTACK
LABEL(nosvs_leave_altstack_end)
#endif
+
+ .globl ibrs_enter, ibrs_enter_end
+ .globl ibrs_leave, ibrs_leave_end
+
+ /* IBRS <- 1 */
+LABEL(ibrs_enter)
+ movl $MSR_IA32_SPEC_CTRL,%ecx
+ movl $IA32_SPEC_CTRL_IBRS,%eax
+ movl $(IA32_SPEC_CTRL_IBRS >> 32),%edx
+ wrmsr
+LABEL(ibrs_enter_end)
+
+ /* IBRS <- 0 */
+LABEL(ibrs_leave)
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1234f
+ movl $MSR_IA32_SPEC_CTRL,%ecx
+ xorl %eax,%eax
+ xorl %edx,%edx
+ wrmsr
+1234:
+LABEL(ibrs_leave_end)
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.37 src/sys/arch/amd64/include/frameasm.h:1.38
--- src/sys/arch/amd64/include/frameasm.h:1.37 Sun Feb 25 13:14:27 2018
+++ src/sys/arch/amd64/include/frameasm.h Wed Mar 28 16:02:49 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.37 2018/02/25 13:14:27 maxv Exp $ */
+/* $NetBSD: frameasm.h,v 1.38 2018/03/28 16:02:49 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -44,6 +44,8 @@
#define HP_NAME_SVS_LEAVE 6
#define HP_NAME_SVS_ENTER_ALT 7
#define HP_NAME_SVS_LEAVE_ALT 8
+#define HP_NAME_IBRS_ENTER 9
+#define HP_NAME_IBRS_LEAVE 10
#define HOTPATCH(name, size) \
123: ; \
@@ -61,6 +63,26 @@
HOTPATCH(HP_NAME_STAC, 3) ; \
.byte 0x0F, 0x1F, 0x00 ; \
+/*
+ * IBRS
+ */
+
+#define IBRS_ENTER_BYTES 17
+#define IBRS_ENTER \
+ HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \
+ NOIBRS_ENTER
+#define NOIBRS_ENTER \
+ .byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \
+ .fill (IBRS_ENTER_BYTES-2),1,0xCC
+
+#define IBRS_LEAVE_BYTES 21
+#define IBRS_LEAVE \
+ HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \
+ NOIBRS_LEAVE
+#define NOIBRS_LEAVE \
+ .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \
+ .fill (IBRS_LEAVE_BYTES-2),1,0xCC
+
#define SWAPGS NOT_XEN(swapgs)
/*
@@ -158,6 +180,7 @@
testb $SEL_UPL,TF_CS(%rsp) ; \
je 98f ; \
SWAPGS ; \
+ IBRS_ENTER ; \
SVS_ENTER ; \
movw %gs,TF_GS(%rsp) ; \
movw %fs,TF_FS(%rsp) ; \
Index: src/sys/arch/x86/x86/spectre.c
diff -u src/sys/arch/x86/x86/spectre.c:1.1 src/sys/arch/x86/x86/spectre.c:1.2
--- src/sys/arch/x86/x86/spectre.c:1.1 Wed Mar 28 14:56:59 2018
+++ src/sys/arch/x86/x86/spectre.c Wed Mar 28 16:02:49 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $ */
+/* $NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $ */
/*
* Copyright (c) 2018 NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.1 2018/03/28 14:56:59 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.2 2018/03/28 16:02:49 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -45,6 +45,7 @@ __KERNEL_RCSID(0, "$NetBSD: spectre.c,v
#include <machine/cpufunc.h>
#include <machine/cpuvar.h>
#include <machine/specialreg.h>
+#include <machine/frameasm.h>
#include <x86/cputypes.h>
@@ -57,13 +58,46 @@ enum spec_mitigation {
bool spec_mitigation_enabled __read_mostly = false;
static enum spec_mitigation mitigation_method = MITIGATION_NONE;
+void speculation_barrier(struct lwp *, struct lwp *);
+
+void
+speculation_barrier(struct lwp *oldlwp, struct lwp *newlwp)
+{
+ if (!spec_mitigation_enabled)
+ return;
+
+ /*
+ * From kernel thread to kernel thread, no need for a barrier.
+ */
+ if ((oldlwp->l_flag & LW_SYSTEM) &&
+ (newlwp->l_flag & LW_SYSTEM))
+ return;
+
+ switch (mitigation_method) {
+ case MITIGATION_INTEL_IBRS:
+ wrmsr(MSR_IA32_PRED_CMD, IA32_PRED_CMD_IBPB);
+ break;
+ default:
+ /* nothing */
+ break;
+ }
+}
+
static void
speculation_detect_method(void)
{
struct cpu_info *ci = curcpu();
+ u_int descs[4];
if (cpu_vendor == CPUVENDOR_INTEL) {
- /* TODO: detect MITIGATION_INTEL_IBRS */
+ if (cpuid_level >= 7) {
+ x86_cpuid(7, descs);
+ if (descs[3] & CPUID_SEF_IBRS) {
+ /* descs[3] = %edx */
+ mitigation_method = MITIGATION_INTEL_IBRS;
+ return;
+ }
+ }
mitigation_method = MITIGATION_NONE;
} else if (cpu_vendor == CPUVENDOR_AMD) {
/*
@@ -88,6 +122,118 @@ speculation_detect_method(void)
}
}
+/* -------------------------------------------------------------------------- */
+
+#ifdef __x86_64__
+static volatile unsigned long ibrs_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long ibrs_cpu_barrier2 __cacheline_aligned;
+
+static void
+ibrs_enable_hotpatch(void)
+{
+ extern uint8_t ibrs_enter, ibrs_enter_end;
+ extern uint8_t ibrs_leave, ibrs_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &ibrs_enter;
+ size = (size_t)&ibrs_enter_end - (size_t)&ibrs_enter;
+ x86_hotpatch(HP_NAME_IBRS_ENTER, bytes, size);
+
+ bytes = &ibrs_leave;
+ size = (size_t)&ibrs_leave_end - (size_t)&ibrs_leave;
+ x86_hotpatch(HP_NAME_IBRS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+
+static void
+ibrs_change_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ bool enabled = (bool)arg1;
+ u_long psl;
+
+ psl = x86_read_psl();
+ x86_disable_intr();
+
+ atomic_dec_ulong(&ibrs_cpu_barrier1);
+ while (atomic_cas_ulong(&ibrs_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* cpu0 is the one that does the hotpatch job */
+ if (ci == &cpu_info_primary) {
+ if (enabled) {
+ ibrs_enable_hotpatch();
+ } else {
+ /* TODO */
+ }
+ }
+
+ atomic_dec_ulong(&ibrs_cpu_barrier2);
+ while (atomic_cas_ulong(&ibrs_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* Write back and invalidate cache, flush pipelines. */
+ wbinvd();
+ x86_flush();
+
+ x86_write_psl(psl);
+}
+
+static int
+ibrs_change(bool enabled)
+{
+ struct cpu_info *ci = NULL;
+ CPU_INFO_ITERATOR cii;
+ uint64_t xc;
+
+ mutex_enter(&cpu_lock);
+
+ /*
+ * We expect all the CPUs to be online.
+ */
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ struct schedstate_percpu *spc = &ci->ci_schedstate;
+ if (spc->spc_flags & SPCF_OFFLINE) {
+ printf("[!] cpu%d offline, IBRS not changed\n",
+ cpu_index(ci));
+ mutex_exit(&cpu_lock);
+ return EOPNOTSUPP;
+ }
+ }
+
+ ibrs_cpu_barrier1 = ncpu;
+ ibrs_cpu_barrier2 = ncpu;
+
+ printf("[+] %s SpectreV2 Mitigation (IBRS)...",
+ (enabled == true) ? "Enabling" : "Disabling");
+ xc = xc_broadcast(0, ibrs_change_cpu, (void *)enabled, NULL);
+ xc_wait(xc);
+ printf(" done!\n");
+
+ mutex_exit(&cpu_lock);
+
+ return 0;
+}
+#else
+/*
+ * TODO: i386
+ */
+static int
+ibrs_change(bool enabled)
+{
+ panic("not supported");
+}
+#endif
+
+/* -------------------------------------------------------------------------- */
+
static void
mitigation_disable_cpu(void *arg1, void *arg2)
{
@@ -95,6 +241,7 @@ mitigation_disable_cpu(void *arg1, void
switch (mitigation_method) {
case MITIGATION_NONE:
+ case MITIGATION_INTEL_IBRS:
panic("impossible");
break;
case MITIGATION_AMD_DIS_IND:
@@ -102,9 +249,6 @@ mitigation_disable_cpu(void *arg1, void
msr &= ~IC_CFG_DIS_IND;
wrmsr(MSR_IC_CFG, msr);
break;
- case MITIGATION_INTEL_IBRS:
- /* ibrs_disable() TODO */
- break;
}
}
@@ -115,6 +259,7 @@ mitigation_enable_cpu(void *arg1, void *
switch (mitigation_method) {
case MITIGATION_NONE:
+ case MITIGATION_INTEL_IBRS:
panic("impossible");
break;
case MITIGATION_AMD_DIS_IND:
@@ -122,9 +267,6 @@ mitigation_enable_cpu(void *arg1, void *
msr |= IC_CFG_DIS_IND;
wrmsr(MSR_IC_CFG, msr);
break;
- case MITIGATION_INTEL_IBRS:
- /* ibrs_enable() TODO */
- break;
}
}
@@ -132,6 +274,7 @@ static int
mitigation_disable(void)
{
uint64_t xc;
+ int error;
speculation_detect_method();
@@ -148,7 +291,10 @@ mitigation_disable(void)
spec_mitigation_enabled = false;
return 0;
case MITIGATION_INTEL_IBRS:
- /* TODO */
+ error = ibrs_change(false);
+ if (error)
+ return error;
+ spec_mitigation_enabled = false;
return 0;
default:
panic("impossible");
@@ -159,6 +305,7 @@ static int
mitigation_enable(void)
{
uint64_t xc;
+ int error;
speculation_detect_method();
@@ -175,7 +322,10 @@ mitigation_enable(void)
spec_mitigation_enabled = true;
return 0;
case MITIGATION_INTEL_IBRS:
- /* TODO */
+ error = ibrs_change(true);
+ if (error)
+ return error;
+ spec_mitigation_enabled = true;
return 0;
default:
panic("impossible");
