Module Name: src
Committed By: dsl
Date: Tue Feb 25 22:16:52 UTC 2014
Modified Files:
src/sys/arch/i386/i386: cpufunc.S
src/sys/arch/x86/include: cpu_extended_state.h cpufunc.h fpu.h
src/sys/arch/x86/x86: cpu.c fpu.c vm_machdep.c
Log Message:
Add support for saving the AVX-256 ymm registers during FPU context switches.
Add support for the forthcoming AVX-512 registers.
Code compiled with -mavx seems to work, but I've not tested context
switches with live ymm registers.
There is a small cost on fork/exec (a larger area is copied/zerod),
but I don't think the ymm registers are read/written unless they
have been used.
The code use XSAVE on all cpus, I'm not brave enough to enable XSAVEOPT.
To generate a diff of this commit:
cvs rdiff -u -r1.18 -r1.19 src/sys/arch/i386/i386/cpufunc.S
cvs rdiff -u -r1.8 -r1.9 src/sys/arch/x86/include/cpu_extended_state.h
cvs rdiff -u -r1.17 -r1.18 src/sys/arch/x86/include/cpufunc.h
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/x86/include/fpu.h
cvs rdiff -u -r1.109 -r1.110 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.8 -r1.9 src/sys/arch/x86/x86/fpu.c
cvs rdiff -u -r1.23 -r1.24 src/sys/arch/x86/x86/vm_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/i386/i386/cpufunc.S
diff -u src/sys/arch/i386/i386/cpufunc.S:1.18 src/sys/arch/i386/i386/cpufunc.S:1.19
--- src/sys/arch/i386/i386/cpufunc.S:1.18 Wed Feb 12 23:24:09 2014
+++ src/sys/arch/i386/i386/cpufunc.S Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.18 2014/02/12 23:24:09 dsl Exp $ */
+/* $NetBSD: cpufunc.S,v 1.19 2014/02/25 22:16:52 dsl Exp $ */
/*-
* Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
#include <sys/errno.h>
#include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.18 2014/02/12 23:24:09 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpufunc.S,v 1.19 2014/02/25 22:16:52 dsl Exp $");
#include "opt_xen.h"
@@ -160,6 +160,24 @@ ENTRY(rdmsr_safe)
movl %eax, PCB_ONFAULT(%ecx)
ret
+END(rdmsr_safe)
+
+/* uint64_t rdxcr(uint32_t) */
+ENTRY(rdxcr)
+ movl 4(%esp), %ecx /* extended control reg number */
+ xgetbv /* Read to %edx:%eax */
+ ret
+END(rdxcr)
+
+/* void wrxcr(uint32_t, uint64_t) */
+ENTRY(wrxcr)
+ movl 4(%esp), %ecx /* extended control reg number */
+ movl 8(%esp), %eax /* feature mask bits */
+ movl 12(%esp), %edx
+ xsetbv
+ ret
+END(wrxcr)
+
/*
* MSR operations fault handler
@@ -389,15 +407,41 @@ ENTRY(fxrstor)
ret
END(fxrstor)
+ENTRY(xsave)
+ movl 4(%esp), %ecx
+ movl 8(%esp), %eax /* feature mask bits */
+ movl 12(%esp), %edx
+ xsave (%ecx)
+ ret
+END(xsave)
+
+ENTRY(xsaveopt)
+ movl 4(%esp), %ecx
+ movl 8(%esp), %eax /* feature mask bits */
+ movl 12(%esp), %edx
+ xsaveopt (%ecx)
+ ret
+END(xsaveopt)
+
+ENTRY(xrstor)
+ movl 4(%esp), %ecx
+ movl 8(%esp), %eax /* feature mask bits */
+ movl 12(%esp), %edx
+ xrstor (%eax)
+ ret
+END(xrstor)
+
ENTRY(x86_stmxcsr)
movl 4(%esp), %eax
stmxcsr (%eax)
ret
+END(x86_stmxcsr)
ENTRY(x86_ldmxcsr)
movl 4(%esp), %eax
ldmxcsr (%eax)
ret
+END(x86_ldmxcsr)
ENTRY(fldummy)
ffree %st(7)
Index: src/sys/arch/x86/include/cpu_extended_state.h
diff -u src/sys/arch/x86/include/cpu_extended_state.h:1.8 src/sys/arch/x86/include/cpu_extended_state.h:1.9
--- src/sys/arch/x86/include/cpu_extended_state.h:1.8 Tue Feb 18 18:39:10 2014
+++ src/sys/arch/x86/include/cpu_extended_state.h Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu_extended_state.h,v 1.8 2014/02/18 18:39:10 dsl Exp $ */
+/* $NetBSD: cpu_extended_state.h,v 1.9 2014/02/25 22:16:52 dsl Exp $ */
#ifndef _X86_CPU_EXTENDED_STATE_H_
#define _X86_CPU_EXTENDED_STATE_H_
@@ -122,19 +122,13 @@ struct fxsave_os {
uint16_t fxo_dflt_cw; /* Control word for signal handlers */
};
-union savefpu {
- struct save87 sv_87;
- struct fxsave sv_xmm;
- struct fxsave_os sv_os;
-};
-
/*
- * For XSAVE a 64byte header follows the above.
+ * For XSAVE a 64byte header follows the fxsave data.
* Currently it only contains one field of which only 3 bits are defined.
* Some other parts must be zero - zero it all.
*
* The xsh_xstate_bv bits match those of XCR0:
- * XCR0_X87 0x00000001 x87 FPU/MMX state (always set)
+ * XCR0_X87 0x00000001 x87 FPU/MMX state
* XCR0_SSE 0x00000002 SSE state
* XCR0_AVX 0x00000004 AVX state (ymmn registers)
*
@@ -143,11 +137,12 @@ union savefpu {
*/
struct xsave_header {
+ uint64_t xsh_fxsave[64]; /* to align in the union */
uint64_t xsh_xstate_bv; /* bitmap of saved sub structures */
uint64_t xsh_rsrvd[2]; /* must be zero */
uint64_t xsh_reserved[5];/* best if zero */
};
-__CTASSERT(sizeof (struct xsave_header) == 64);
+__CTASSERT(sizeof (struct xsave_header) == 512 + 64);
/*
* The ymm save area actually follows the xsave_header.
@@ -157,6 +152,20 @@ struct xsave_ymm {
};
__CTASSERT(sizeof (struct xsave_ymm) == 256);
+/*
+ * The following union is placed at the end of the pcb.
+ * It is defined this way to separate the definitions and to
+ * minimise the number of union/struct selectors.
+ * NB: Some userspace stuff (eg firefox) uses it to parse ucontext.
+ */
+union savefpu {
+ struct save87 sv_87;
+ struct fxsave sv_xmm;
+#ifdef _KERNEL
+ struct fxsave_os sv_os;
+ struct xsave_header sv_xsave_hdr;
+#endif
+};
/*
* 80387 control and status word bits
Index: src/sys/arch/x86/include/cpufunc.h
diff -u src/sys/arch/x86/include/cpufunc.h:1.17 src/sys/arch/x86/include/cpufunc.h:1.18
--- src/sys/arch/x86/include/cpufunc.h:1.17 Thu Feb 13 19:37:08 2014
+++ src/sys/arch/x86/include/cpufunc.h Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.17 2014/02/13 19:37:08 dsl Exp $ */
+/* $NetBSD: cpufunc.h,v 1.18 2014/02/25 22:16:52 dsl Exp $ */
/*-
* Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -77,27 +77,36 @@ void x86_hlt(void);
void x86_stihlt(void);
u_int x86_getss(void);
-struct save87;
-struct fxsave;
+/* fpu save, restore etc */
+union savefpu;
void fldcw(const uint16_t *);
void fnclex(void);
void fninit(void);
-void fnsave(struct save87 *);
+void fnsave(union savefpu *);
void fnstcw(uint16_t *);
uint16_t fngetsw(void);
void fnstsw(uint16_t *);
-void frstor(const struct save87 *);
+void frstor(const union savefpu *);
void fwait(void);
void clts(void);
void stts(void);
-void fxsave(struct fxsave *);
-void fxrstor(const struct fxsave *);
+void fxsave(union savefpu *);
+void fxrstor(const union savefpu *);
void x86_ldmxcsr(const uint32_t *);
void x86_stmxcsr(uint32_t *);
void fldummy(void);
void fp_divide_by_0(void);
+/* Extended processor state functions (for AVX registers etc) */
+
+uint64_t rdxcr(uint32_t); /* xgetbv */
+void wrxcr(uint32_t, uint64_t); /* xsetgv */
+
+void xrstor(const union savefpu *, uint64_t);
+void xsave(union savefpu *, uint64_t);
+void xsaveopt(union savefpu *, uint64_t);
+
void x86_monitor(const void *, uint32_t, uint32_t);
void x86_mwait(uint32_t, uint32_t);
/* x86_cpuid2() writes four 32bit values, %eax, %ebx, %ecx and %edx */
@@ -134,14 +143,6 @@ void wrmsr_locked(u_int, u_int, uint64_
void setfs(int);
void setusergs(int);
-/* Extended processor state functions (for AVX registers etc) */
-
-uint64_t rdxcr(uint32_t); /* xgetbv */
-void wrxcr(uint32_t, uint64_t); /* xsetgv */
-void xrstor(const void *, uint64_t);
-void xsave(void *, uint64_t);
-void xsaveopt(const void *, uint64_t);
-
#endif /* _KERNEL */
#endif /* !_X86_CPUFUNC_H_ */
Index: src/sys/arch/x86/include/fpu.h
diff -u src/sys/arch/x86/include/fpu.h:1.5 src/sys/arch/x86/include/fpu.h:1.6
--- src/sys/arch/x86/include/fpu.h:1.5 Sun Feb 23 22:35:27 2014
+++ src/sys/arch/x86/include/fpu.h Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: fpu.h,v 1.5 2014/02/23 22:35:27 dsl Exp $ */
+/* $NetBSD: fpu.h,v 1.6 2014/02/25 22:16:52 dsl Exp $ */
#ifndef _X86_FPU_H_
#define _X86_FPU_H_
@@ -28,6 +28,9 @@ void fpu_save_area_clear(struct lwp *, u
/* Reset control words only - for signal handlers */
void fpu_save_area_reset(struct lwp *);
+/* Copy data outside pcb during fork */
+void fpu_save_area_fork(struct pcb *, const struct pcb *);
+
/* Load FP registers with user-supplied values */
void process_write_fpregs_xmm(struct lwp *lwp, const struct fxsave *fpregs);
void process_write_fpregs_s87(struct lwp *lwp, const struct save87 *fpregs);
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.109 src/sys/arch/x86/x86/cpu.c:1.110
--- src/sys/arch/x86/x86/cpu.c:1.109 Wed Feb 19 21:23:02 2014
+++ src/sys/arch/x86/x86/cpu.c Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.109 2014/02/19 21:23:02 dsl Exp $ */
+/* $NetBSD: cpu.c,v 1.110 2014/02/25 22:16:52 dsl Exp $ */
/*-
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.109 2014/02/19 21:23:02 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.110 2014/02/25 22:16:52 dsl Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -552,29 +552,41 @@ cpu_childdetached(device_t self, device_
void
cpu_init(struct cpu_info *ci)
{
+ uint32_t cr4;
lcr0(rcr0() | CR0_WP);
+ cr4 = rcr4();
/*
* On a P6 or above, enable global TLB caching if the
* hardware supports it.
*/
if (cpu_feature[0] & CPUID_PGE)
- lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
+ cr4 |= CR4_PGE; /* enable global TLB caching */
/*
* If we have FXSAVE/FXRESTOR, use them.
*/
if (cpu_feature[0] & CPUID_FXSR) {
- lcr4(rcr4() | CR4_OSFXSR);
+ cr4 |= CR4_OSFXSR;
/*
* If we have SSE/SSE2, enable XMM exceptions.
*/
if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
- lcr4(rcr4() | CR4_OSXMMEXCPT);
+ cr4 |= CR4_OSXMMEXCPT;
}
+ /* If xsave is supported, enable it */
+ if (cpu_feature[1] & CPUID2_XSAVE)
+ cr4 |= CR4_OSXSAVE;
+
+ lcr4(cr4);
+
+ /* If xsave is enabled, enable all fpu features */
+ if (cr4 & CR4_OSXSAVE)
+ wrxcr(0, x86_xsave_features & XCR0_FPU);
+
#ifdef MTRR
/*
* On a P6 or above, initialize MTRR's if the hardware supports them.
Index: src/sys/arch/x86/x86/fpu.c
diff -u src/sys/arch/x86/x86/fpu.c:1.8 src/sys/arch/x86/x86/fpu.c:1.9
--- src/sys/arch/x86/x86/fpu.c:1.8 Sun Feb 23 22:35:28 2014
+++ src/sys/arch/x86/x86/fpu.c Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: fpu.c,v 1.8 2014/02/23 22:35:28 dsl Exp $ */
+/* $NetBSD: fpu.c,v 1.9 2014/02/25 22:16:52 dsl Exp $ */
/*-
* Copyright (c) 2008 The NetBSD Foundation, Inc. All
@@ -100,7 +100,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.8 2014/02/23 22:35:28 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.9 2014/02/25 22:16:52 dsl Exp $");
#include "opt_multiprocessor.h"
@@ -429,21 +429,26 @@ fpudna(struct trapframe *frame)
pcb->pcb_fpcpu = ci;
if (i386_use_fxsave) {
- /*
- * AMD FPU's do not restore FIP, FDP, and FOP on fxrstor,
- * leaking other process's execution history.
- * Clear them manually by loading a zero onto the fpu stack.
- *
- * Clear the ES bit in the x87 status word if it is currently
- * set, in order to avoid causing a fault in the upcoming load.
- */
- if (fngetsw() & 0x80)
- fnclex();
- fldummy();
+ if (x86_xsave_features != 0) {
+ xrstor(&pcb->pcb_savefpu, x86_xsave_features);
+ } else {
+ /*
+ * AMD FPU's do not restore FIP, FDP, and FOP on
+ * fxrstor, leaking other process's execution history.
+ * Clear them manually by loading a zero.
+ *
+ * Clear the ES bit in the x87 status word if it is
+ * currently set, in order to avoid causing a fault
+ * in the upcoming load.
+ */
+ if (fngetsw() & 0x80)
+ fnclex();
+ fldummy();
- fxrstor(&pcb->pcb_savefpu.sv_xmm);
+ fxrstor(&pcb->pcb_savefpu);
+ }
} else {
- frstor(&pcb->pcb_savefpu.sv_87);
+ frstor(&pcb->pcb_savefpu);
}
KASSERT(ci == curcpu());
@@ -472,9 +477,12 @@ fpusave_cpu(bool save)
if (save) {
clts();
if (i386_use_fxsave) {
- fxsave(&pcb->pcb_savefpu.sv_xmm);
+ if (x86_xsave_features != 0)
+ xsave(&pcb->pcb_savefpu, x86_xsave_features);
+ else
+ fxsave(&pcb->pcb_savefpu);
} else {
- fnsave(&pcb->pcb_savefpu.sv_87);
+ fnsave(&pcb->pcb_savefpu);
}
}
@@ -533,7 +541,7 @@ fpusave_lwp(struct lwp *l, bool save)
/*
* exec needs to clear the fpu save area to avoid leaking info from the
* old process to userspace.
- * We must also load these values into the fpu - otherwise the process
+ * We must also (later) load these values into the fpu - otherwise the process
* will see another processes fpu registers.
*/
void
@@ -551,7 +559,7 @@ fpu_save_area_clear(struct lwp *lwp, uns
fpu_save->sv_xmm.fx_mxcsr_mask = __INITIAL_MXCSR_MASK__;
fpu_save->sv_xmm.fx_cw = x87_cw;
} else {
- memset(&fpu_save->sv_87, 0, sizeof fpu_save->sv_87);
+ memset(&fpu_save->sv_87, 0, x86_fpu_save_size);
fpu_save->sv_87.s87_tw = 0xffff;
fpu_save->sv_87.s87_cw = x87_cw;
}
@@ -575,6 +583,23 @@ fpu_save_area_reset(struct lwp *lwp)
}
}
+/* During fork the xsave data needs to be copied */
+void
+fpu_save_area_fork(struct pcb *pcb2, const struct pcb *pcb1)
+{
+ ssize_t extra;
+
+ /* The pcb itself has been copied, but the xsave area
+ * extends further. */
+
+ extra = offsetof(struct pcb, pcb_savefpu) + x86_fpu_save_size -
+ sizeof (struct pcb);
+
+ if (extra > 0)
+ memcpy(pcb2 + 1, pcb1 + 1, extra);
+}
+
+
/*
* Write the FP registers.
* Buffer has usually come from userspace so should not be trusted.
@@ -621,6 +646,7 @@ void
process_read_fpregs_xmm(struct lwp *lwp, struct fxsave *fpregs)
{
fpusave_lwp(lwp, true);
+
if (i386_use_fxsave) {
memcpy(fpregs, &process_fpframe(lwp)->sv_xmm,
sizeof process_fpframe(lwp)->sv_xmm);
Index: src/sys/arch/x86/x86/vm_machdep.c
diff -u src/sys/arch/x86/x86/vm_machdep.c:1.23 src/sys/arch/x86/x86/vm_machdep.c:1.24
--- src/sys/arch/x86/x86/vm_machdep.c:1.23 Thu Feb 20 18:19:10 2014
+++ src/sys/arch/x86/x86/vm_machdep.c Tue Feb 25 22:16:52 2014
@@ -1,4 +1,4 @@
-/* $NetBSD: vm_machdep.c,v 1.23 2014/02/20 18:19:10 dsl Exp $ */
+/* $NetBSD: vm_machdep.c,v 1.24 2014/02/25 22:16:52 dsl Exp $ */
/*-
* Copyright (c) 1982, 1986 The Regents of the University of California.
@@ -80,7 +80,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.23 2014/02/20 18:19:10 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.24 2014/02/25 22:16:52 dsl Exp $");
#include "opt_mtrr.h"
@@ -154,6 +154,8 @@ cpu_lwp_fork(struct lwp *l1, struct lwp
/* Copy the PCB from parent. */
memcpy(pcb2, pcb1, sizeof(struct pcb));
+ /* Copy any additional fpu state */
+ fpu_save_area_fork(pcb2, pcb1);
#if defined(XEN)
pcb2->pcb_iopl = SEL_KPL;
