Module Name: src
Committed By: maxv
Date: Tue May 14 16:59:26 UTC 2019
Modified Files:
src/sys/arch/amd64/amd64: amd64_trap.S locore.S
src/sys/arch/amd64/include: frameasm.h
src/sys/arch/x86/include: specialreg.h
src/sys/arch/x86/x86: spectre.c
Log Message:
Mitigation for INTEL-SA-00233: Microarchitectural Data Sampling (MDS).
It requires a microcode update, now available on the Intel website. The
microcode modifies the behavior of the VERW instruction, and makes it flush
internal CPU buffers. We hotpatch the return-to-userland path to add VERW.
Two sysctls are added:
machdep.mds.mitigated = {0/1} user-settable
machdep.mds.method = {string} constructed by the kernel
The kernel will automatically enable the mitigation if the updated
microcode is present. If the new microcode is not present, the user can
load it via cpuctl, and set machdep.mds.mitigated=1.
To generate a diff of this commit:
cvs rdiff -u -r1.46 -r1.47 src/sys/arch/amd64/amd64/amd64_trap.S
cvs rdiff -u -r1.180 -r1.181 src/sys/arch/amd64/amd64/locore.S
cvs rdiff -u -r1.42 -r1.43 src/sys/arch/amd64/include/frameasm.h
cvs rdiff -u -r1.143 -r1.144 src/sys/arch/x86/include/specialreg.h
cvs rdiff -u -r1.26 -r1.27 src/sys/arch/x86/x86/spectre.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/amd64_trap.S
diff -u src/sys/arch/amd64/amd64/amd64_trap.S:1.46 src/sys/arch/amd64/amd64/amd64_trap.S:1.47
--- src/sys/arch/amd64/amd64/amd64_trap.S:1.46 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/amd64/amd64/amd64_trap.S Tue May 14 16:59:25 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: amd64_trap.S,v 1.46 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: amd64_trap.S,v 1.47 2019/05/14 16:59:25 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
@@ -281,6 +281,7 @@ IDTVEC(trap02)
call _C_LABEL(nmitrap)
.Lnmileave:
+ MDS_LEAVE
SVS_LEAVE_NMI
IBRS_LEAVE
INTR_RESTORE_GPRS
@@ -369,6 +370,7 @@ IDTVEC(trap08)
incq CPUVAR(NTRAP)
call _C_LABEL(doubletrap)
+ MDS_LEAVE
SVS_LEAVE_ALTSTACK
IBRS_LEAVE
INTR_RESTORE_GPRS
Index: src/sys/arch/amd64/amd64/locore.S
diff -u src/sys/arch/amd64/amd64/locore.S:1.180 src/sys/arch/amd64/amd64/locore.S:1.181
--- src/sys/arch/amd64/amd64/locore.S:1.180 Sat Mar 9 08:42:25 2019
+++ src/sys/arch/amd64/amd64/locore.S Tue May 14 16:59:25 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: locore.S,v 1.180 2019/03/09 08:42:25 maxv Exp $ */
+/* $NetBSD: locore.S,v 1.181 2019/05/14 16:59:25 maxv Exp $ */
/*
* Copyright-o-rama!
@@ -1483,6 +1483,7 @@ IDTVEC_END(osyscall)
TEXT_USER_BEGIN
_ALIGN_TEXT
LABEL(syscall_sysret)
+ MDS_LEAVE
SVS_LEAVE
IBRS_LEAVE
INTR_RESTORE_GPRS
@@ -1574,6 +1575,7 @@ END(pagezero)
_ALIGN_TEXT
LABEL(intrfastexit)
NOT_XEN(cli;)
+ MDS_LEAVE
SVS_LEAVE
IBRS_LEAVE
INTR_RESTORE_GPRS
@@ -1721,3 +1723,18 @@ LABEL(noibrs_enter_end)
LABEL(noibrs_leave)
NOIBRS_LEAVE
LABEL(noibrs_leave_end)
+
+ .globl mds_leave, mds_leave_end
+
+LABEL(mds_leave)
+ testb $SEL_UPL,TF_CS(%rsp)
+ jz 1234f
+ pushq $GSEL(GDATA_SEL, SEL_KPL)
+ verw (%rsp)
+ addq $8,%rsp
+1234:
+LABEL(mds_leave_end)
+
+LABEL(nomds_leave)
+ NOMDS_LEAVE
+LABEL(nomds_leave_end)
Index: src/sys/arch/amd64/include/frameasm.h
diff -u src/sys/arch/amd64/include/frameasm.h:1.42 src/sys/arch/amd64/include/frameasm.h:1.43
--- src/sys/arch/amd64/include/frameasm.h:1.42 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/amd64/include/frameasm.h Tue May 14 16:59:25 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: frameasm.h,v 1.42 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: frameasm.h,v 1.43 2019/05/14 16:59:25 maxv Exp $ */
#ifndef _AMD64_MACHINE_FRAMEASM_H
#define _AMD64_MACHINE_FRAMEASM_H
@@ -48,6 +48,7 @@
#define HP_NAME_IBRS_LEAVE 10
#define HP_NAME_SVS_ENTER_NMI 11
#define HP_NAME_SVS_LEAVE_NMI 12
+#define HP_NAME_MDS_LEAVE 13
#define HOTPATCH(name, size) \
123: ; \
@@ -85,6 +86,18 @@
.byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \
.fill (IBRS_LEAVE_BYTES-2),1,0xCC
+/*
+ * MDS
+ */
+
+#define MDS_LEAVE_BYTES 20
+#define MDS_LEAVE \
+ HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \
+ NOMDS_LEAVE
+#define NOMDS_LEAVE \
+ .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \
+ .fill (MDS_LEAVE_BYTES-2),1,0xCC
+
#define SWAPGS NOT_XEN(swapgs)
/*
Index: src/sys/arch/x86/include/specialreg.h
diff -u src/sys/arch/x86/include/specialreg.h:1.143 src/sys/arch/x86/include/specialreg.h:1.144
--- src/sys/arch/x86/include/specialreg.h:1.143 Wed Mar 13 05:22:07 2019
+++ src/sys/arch/x86/include/specialreg.h Tue May 14 16:59:26 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: specialreg.h,v 1.143 2019/03/13 05:22:07 msaitoh Exp $ */
+/* $NetBSD: specialreg.h,v 1.144 2019/05/14 16:59:26 maxv Exp $ */
/*-
* Copyright (c) 1991 The Regents of the University of California.
@@ -426,6 +426,7 @@
/* %edx */
#define CPUID_SEF_AVX512_4VNNIW __BIT(2)
#define CPUID_SEF_AVX512_4FMAPS __BIT(3)
+#define CPUID_SEF_MD_CLEAR __BIT(10)
#define CPUID_SEF_TSX_FORCE_ABORT __BIT(13) /* MSR_TSX_FORCE_ABORT bit 0 */
#define CPUID_SEF_IBRS __BIT(26) /* IBRS / IBPB Speculation Control */
#define CPUID_SEF_STIBP __BIT(27) /* STIBP Speculation Control */
@@ -747,6 +748,7 @@
#define IA32_ARCH_RSBA 0x04
#define IA32_ARCH_SKIP_L1DFL_VMENTRY 0x08
#define IA32_ARCH_SSB_NO 0x10
+#define IA32_ARCH_MDS_NO 0x20
#define MSR_IA32_FLUSH_CMD 0x10b
#define IA32_FLUSH_CMD_L1D_FLUSH 0x01
#define MSR_TSX_FORCE_ABORT 0x10f
Index: src/sys/arch/x86/x86/spectre.c
diff -u src/sys/arch/x86/x86/spectre.c:1.26 src/sys/arch/x86/x86/spectre.c:1.27
--- src/sys/arch/x86/x86/spectre.c:1.26 Sat Apr 27 10:40:17 2019
+++ src/sys/arch/x86/x86/spectre.c Tue May 14 16:59:26 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: spectre.c,v 1.26 2019/04/27 10:40:17 maxv Exp $ */
+/* $NetBSD: spectre.c,v 1.27 2019/05/14 16:59:26 maxv Exp $ */
/*
* Copyright (c) 2018 NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.26 2019/04/27 10:40:17 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.27 2019/05/14 16:59:26 maxv Exp $");
#include "opt_spectre.h"
@@ -549,6 +549,226 @@ sysctl_machdep_spectreV4_mitigated(SYSCT
/* -------------------------------------------------------------------------- */
+enum mds_mitigation {
+ MDS_MITIGATION_NONE,
+ MDS_MITIGATION_VERW,
+ MDS_MITIGATION_MDS_NO
+};
+
+static char mds_mitigation_name[64] = "(none)";
+
+static enum mds_mitigation mds_mitigation_method = MDS_MITIGATION_NONE;
+static bool mds_mitigation_enabled __read_mostly = false;
+
+static volatile unsigned long mds_cpu_barrier1 __cacheline_aligned;
+static volatile unsigned long mds_cpu_barrier2 __cacheline_aligned;
+
+#ifdef __x86_64__
+static void
+mds_disable_hotpatch(void)
+{
+ extern uint8_t nomds_leave, nomds_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &nomds_leave;
+ size = (size_t)&nomds_leave_end - (size_t)&nomds_leave;
+ x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+
+static void
+mds_enable_hotpatch(void)
+{
+ extern uint8_t mds_leave, mds_leave_end;
+ u_long psl, cr0;
+ uint8_t *bytes;
+ size_t size;
+
+ x86_patch_window_open(&psl, &cr0);
+
+ bytes = &mds_leave;
+ size = (size_t)&mds_leave_end - (size_t)&mds_leave;
+ x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size);
+
+ x86_patch_window_close(psl, cr0);
+}
+#else
+/* MDS not supported on i386 */
+static void
+mds_disable_hotpatch(void)
+{
+ panic("%s: impossible", __func__);
+}
+static void
+mds_enable_hotpatch(void)
+{
+ panic("%s: impossible", __func__);
+}
+#endif
+
+static void
+mitigation_mds_apply_cpu(struct cpu_info *ci, bool enabled)
+{
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ case MDS_MITIGATION_MDS_NO:
+ panic("impossible");
+ case MDS_MITIGATION_VERW:
+ /* cpu0 is the one that does the hotpatch job */
+ if (ci == &cpu_info_primary) {
+ if (enabled) {
+ mds_enable_hotpatch();
+ } else {
+ mds_disable_hotpatch();
+ }
+ }
+ break;
+ }
+}
+
+static void
+mitigation_mds_change_cpu(void *arg1, void *arg2)
+{
+ struct cpu_info *ci = curcpu();
+ bool enabled = (bool)arg1;
+ u_long psl = 0;
+
+ /* Rendez-vous 1. */
+ psl = x86_read_psl();
+ x86_disable_intr();
+
+ atomic_dec_ulong(&mds_cpu_barrier1);
+ while (atomic_cas_ulong(&mds_cpu_barrier1, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ mitigation_mds_apply_cpu(ci, enabled);
+
+ /* Rendez-vous 2. */
+ atomic_dec_ulong(&mds_cpu_barrier2);
+ while (atomic_cas_ulong(&mds_cpu_barrier2, 0, 0) != 0) {
+ x86_pause();
+ }
+
+ /* Write back and invalidate cache, flush pipelines. */
+ wbinvd();
+ x86_flush();
+
+ x86_write_psl(psl);
+}
+
+static void
+mds_detect_method(void)
+{
+ u_int descs[4];
+ uint64_t msr;
+
+ if (cpu_vendor != CPUVENDOR_INTEL) {
+ mds_mitigation_method = MDS_MITIGATION_MDS_NO;
+ return;
+ }
+
+ x86_cpuid(0x7, descs);
+ if (descs[3] & CPUID_SEF_ARCH_CAP) {
+ msr = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
+ if (msr & IA32_ARCH_MDS_NO) {
+ mds_mitigation_method = MDS_MITIGATION_MDS_NO;
+ return;
+ }
+ }
+
+#ifdef __x86_64__
+ if (descs[3] & CPUID_SEF_MD_CLEAR) {
+ mds_mitigation_method = MDS_MITIGATION_VERW;
+ }
+#endif
+}
+
+static void
+mds_set_name(void)
+{
+ char name[64] = "";
+
+ if (!mds_mitigation_enabled) {
+ strlcat(name, "(none)", sizeof(name));
+ } else {
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ panic("%s: impossible", __func__);
+ case MDS_MITIGATION_MDS_NO:
+ strlcat(name, "[MDS_NO]", sizeof(name));
+ break;
+ case MDS_MITIGATION_VERW:
+ strlcat(name, "[VERW]", sizeof(name));
+ break;
+ }
+ }
+
+ strlcpy(mds_mitigation_name, name,
+ sizeof(mds_mitigation_name));
+}
+
+static int
+mitigation_mds_change(bool enabled)
+{
+ uint64_t xc;
+
+ mds_detect_method();
+
+ switch (mds_mitigation_method) {
+ case MDS_MITIGATION_NONE:
+ printf("[!] No mitigation available\n");
+ return EOPNOTSUPP;
+ case MDS_MITIGATION_VERW:
+ /* Initialize the barriers */
+ mds_cpu_barrier1 = ncpu;
+ mds_cpu_barrier2 = ncpu;
+
+ printf("[+] %s MDS Mitigation...",
+ enabled ? "Enabling" : "Disabling");
+ xc = xc_broadcast(XC_HIGHPRI, mitigation_mds_change_cpu,
+ (void *)enabled, NULL);
+ xc_wait(xc);
+ printf(" done!\n");
+ mds_mitigation_enabled = enabled;
+ mds_set_name();
+ return 0;
+ case MDS_MITIGATION_MDS_NO:
+ printf("[+] The CPU is not affected by MDS\n");
+ return 0;
+ default:
+ panic("impossible");
+ }
+}
+
+static int
+sysctl_machdep_mds_mitigated(SYSCTLFN_ARGS)
+{
+ struct sysctlnode node;
+ int error;
+ bool val;
+
+ val = *(bool *)rnode->sysctl_data;
+
+ node = *rnode;
+ node.sysctl_data = &val;
+
+ error = sysctl_lookup(SYSCTLFN_CALL(&node));
+ if (error != 0 || newp == NULL)
+ return error;
+
+ if (val == mds_mitigation_enabled)
+ return 0;
+ return mitigation_mds_change(val);
+}
+
+/* -------------------------------------------------------------------------- */
+
void speculation_barrier(struct lwp *, struct lwp *);
void
@@ -628,6 +848,23 @@ cpu_speculation_init(struct cpu_info *ci
}
}
#endif
+
+ /*
+ * Microarchectural Data Sampling.
+ *
+ * cpu0 is the one that detects the method and sets the global
+ * variable.
+ */
+ if (ci == &cpu_info_primary) {
+ mds_detect_method();
+ mds_mitigation_enabled =
+ (mds_mitigation_method != MDS_MITIGATION_NONE);
+ mds_set_name();
+ }
+ if (mds_mitigation_method != MDS_MITIGATION_NONE &&
+ mds_mitigation_method != MDS_MITIGATION_MDS_NO) {
+ mitigation_mds_apply_cpu(ci, true);
+ }
}
void sysctl_speculation_init(struct sysctllog **);
@@ -705,4 +942,26 @@ sysctl_speculation_init(struct sysctllog
NULL, 0,
v4_mitigation_name, 0,
CTL_CREATE, CTL_EOL);
+
+ /* Microarchitectural Data Sampling */
+ spec_rnode = NULL;
+ sysctl_createv(clog, 0, NULL, &spec_rnode,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_NODE, "mds", NULL,
+ NULL, 0, NULL, 0,
+ CTL_MACHDEP, CTL_CREATE);
+ sysctl_createv(clog, 0, &spec_rnode, NULL,
+ CTLFLAG_READWRITE,
+ CTLTYPE_BOOL, "mitigated",
+ SYSCTL_DESCR("Whether MDS is mitigated"),
+ sysctl_machdep_mds_mitigated, 0,
+ &mds_mitigation_enabled, 0,
+ CTL_CREATE, CTL_EOL);
+ sysctl_createv(clog, 0, &spec_rnode, NULL,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_STRING, "method",
+ SYSCTL_DESCR("Mitigation method in use"),
+ NULL, 0,
+ mds_mitigation_name, 0,
+ CTL_CREATE, CTL_EOL);
}
