Author: andrew Date: Thu Oct 20 09:22:10 2016 New Revision: 307667 URL: https://svnweb.freebsd.org/changeset/base/307667
Log: Add support for the fpu_kern(9) KPI on arm64. It hooks into the existing VFP code to store the old context, with lazy loading of the new context when needed. FPU_KERN_NOCTX is missing as this is unused in the crypto code this has been tested with, and I am unsure on the requirements of the UEFI Runtime Services. Reviewed by: kib Obtained from: ABT Systeems Ltd Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D8276 Modified: head/share/man/man9/fpu_kern.9 head/sys/arm64/arm64/trap.c head/sys/arm64/arm64/vfp.c head/sys/arm64/include/pcb.h head/sys/arm64/include/vfp.h head/sys/opencrypto/crypto.c Modified: head/share/man/man9/fpu_kern.9 ============================================================================== --- head/share/man/man9/fpu_kern.9 Thu Oct 20 09:06:39 2016 (r307666) +++ head/share/man/man9/fpu_kern.9 Thu Oct 20 09:22:10 2016 (r307667) @@ -23,7 +23,7 @@ .\" .\" $FreeBSD$ .\" -.Dd October 23, 2014 +.Dd October 20, 2016 .Dt FPU_KERN 9 .Os .Sh NAME @@ -134,11 +134,11 @@ of increased system latency. .El .Pp The function does not sleep or block. -It could cause the +It could cause an FPU trap during execution, and on the first FPU access +after the function returns, as well as after each context switch. +On i386 and amd64 this will be the .Nm Device Not Available -exception during execution, and on the first FPU access after the -function returns, as well as after each context switch -(see Intel Software Developer Manual for the reference). +exception (see Intel Software Developer Manual for the reference). Currently, no errors are defined which can be returned by .Fn fpu_kern_enter to the caller. @@ -190,7 +190,7 @@ and false otherwise. .Sh NOTES The .Nm -is currently implemented only for the i386 and amd64 architectures. +is currently implemented only for the i386, amd64, and arm64 architectures. .Pp There is no way to handle floating point exceptions raised from kernel mode. @@ -208,3 +208,5 @@ The .Nm facitily and this manual page were written by .An Konstantin Belousov Aq Mt k...@freebsd.org . +The arm64 support was added by +.An Andrew Turner Aq Mt and...@freebsd.org . Modified: head/sys/arm64/arm64/trap.c ============================================================================== --- head/sys/arm64/arm64/trap.c Thu Oct 20 09:06:39 2016 (r307666) +++ head/sys/arm64/arm64/trap.c Thu Oct 20 09:22:10 2016 (r307667) @@ -282,9 +282,17 @@ do_el1h_sync(struct trapframe *frame) switch(exception) { case EXCP_FP_SIMD: case EXCP_TRAP_FP: - print_registers(frame); - printf(" esr: %.8lx\n", esr); - panic("VFP exception in the kernel"); +#ifdef VFP + if ((curthread->td_pcb->pcb_fpflags & PCB_FP_KERN) != 0) { + vfp_restore_state(); + } else +#endif + { + print_registers(frame); + printf(" esr: %.8lx\n", esr); + panic("VFP exception in the kernel"); + } + break; case EXCP_INSN_ABORT: case EXCP_DATA_ABORT: far = READ_SPECIALREG(far_el1); @@ -412,6 +420,9 @@ do_el0_sync(struct trapframe *frame) KASSERT((curthread->td_pcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, ("Kernel VFP flags set while entering userspace")); + KASSERT( + curthread->td_pcb->pcb_fpusaved == &curthread->td_pcb->pcb_fpustate, + ("Kernel VFP state in use when entering userspace")); } void Modified: head/sys/arm64/arm64/vfp.c ============================================================================== --- head/sys/arm64/arm64/vfp.c Thu Oct 20 09:06:39 2016 (r307666) +++ head/sys/arm64/arm64/vfp.c Thu Oct 20 09:22:10 2016 (r307667) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 The FreeBSD Foundation + * Copyright (c) 2015-2016 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under @@ -48,6 +48,14 @@ CTASSERT(sizeof(((struct pcb *)0)->pcb_f static MALLOC_DEFINE(M_FPUKERN_CTX, "fpukern_ctx", "Kernel contexts for VFP state"); +struct fpu_kern_ctx { + struct vfpstate *prev; +#define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */ +#define FPU_KERN_CTX_INUSE 0x02 + uint32_t flags; + struct vfpstate state; +}; + static void vfp_enable(void) { @@ -71,9 +79,10 @@ vfp_disable(void) } /* - * Called when the thread is dying. If the thread was the last to use the - * VFP unit mark it as unused to tell the kernel the fp state is unowned. - * Ensure the VFP unit is off so we get an exception on the next access. + * Called when the thread is dying or when discarding the kernel VFP state. + * If the thread was the last to use the VFP unit mark it as unused to tell + * the kernel the fp state is unowned. Ensure the VFP unit is off so we get + * an exception on the next access. */ void vfp_discard(struct thread *td) @@ -226,4 +235,111 @@ vfp_init(void) SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL); +struct fpu_kern_ctx * +fpu_kern_alloc_ctx(u_int flags) +{ + struct fpu_kern_ctx *res; + size_t sz; + + sz = sizeof(struct fpu_kern_ctx); + res = malloc(sz, M_FPUKERN_CTX, ((flags & FPU_KERN_NOWAIT) ? + M_NOWAIT : M_WAITOK) | M_ZERO); + return (res); +} + +void +fpu_kern_free_ctx(struct fpu_kern_ctx *ctx) +{ + + KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("free'ing inuse ctx")); + /* XXXAndrew clear the memory ? */ + free(ctx, M_FPUKERN_CTX); +} + +int +fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, + ("using inuse ctx")); + + if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { + ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; + return (0); + } + /* + * Check either we are already using the VFP in the kernel, or + * the the saved state points to the default user space. + */ + KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0 || + pcb->pcb_fpusaved == &pcb->pcb_fpustate, + ("Mangled pcb_fpusaved %x %p %p", pcb->pcb_fpflags, pcb->pcb_fpusaved, &pcb->pcb_fpustate)); + ctx->flags = FPU_KERN_CTX_INUSE; + vfp_save_state(curthread, pcb); + ctx->prev = pcb->pcb_fpusaved; + pcb->pcb_fpusaved = &ctx->state; + pcb->pcb_fpflags |= PCB_FP_KERN; + pcb->pcb_fpflags &= ~PCB_FP_STARTED; + + return (0); +} + +int +fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + + KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, + ("FPU context not inuse")); + ctx->flags &= ~FPU_KERN_CTX_INUSE; + + if (is_fpu_kern_thread(0) && + (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) + return (0); + KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); + critical_enter(); + vfp_discard(td); + critical_exit(); + pcb->pcb_fpflags &= ~PCB_FP_STARTED; + pcb->pcb_fpusaved = ctx->prev; + + if (pcb->pcb_fpusaved == &pcb->pcb_fpustate) { + pcb->pcb_fpflags &= ~PCB_FP_KERN; + } else { + KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0, + ("unpaired fpu_kern_leave")); + } + + return (0); +} + +int +fpu_kern_thread(u_int flags) +{ + struct pcb *pcb = curthread->td_pcb; + + KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, + ("Only kthread may use fpu_kern_thread")); + KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, + ("Mangled pcb_fpusaved")); + KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) == 0, + ("Thread already setup for the VFP")); + pcb->pcb_fpflags |= PCB_FP_KERN; + return (0); +} + +int +is_fpu_kern_thread(u_int flags) +{ + struct pcb *curpcb; + + if ((curthread->td_pflags & TDP_KTHREAD) == 0) + return (0); + curpcb = curthread->td_pcb; + return ((curpcb->pcb_fpflags & PCB_FP_KERN) != 0); +} #endif Modified: head/sys/arm64/include/pcb.h ============================================================================== --- head/sys/arm64/include/pcb.h Thu Oct 20 09:06:39 2016 (r307666) +++ head/sys/arm64/include/pcb.h Thu Oct 20 09:22:10 2016 (r307667) @@ -54,6 +54,7 @@ struct pcb { struct vfpstate *pcb_fpusaved; int pcb_fpflags; #define PCB_FP_STARTED 0x01 +#define PCB_FP_KERN 0x02 /* The bits passed to userspace in get_fpcontext */ #define PCB_FP_USERMASK (PCB_FP_STARTED) u_int pcb_vfpcpu; /* Last cpu this thread ran VFP code */ Modified: head/sys/arm64/include/vfp.h ============================================================================== --- head/sys/arm64/include/vfp.h Thu Oct 20 09:06:39 2016 (r307666) +++ head/sys/arm64/include/vfp.h Thu Oct 20 09:22:10 2016 (r307667) @@ -45,6 +45,23 @@ void vfp_init(void); void vfp_discard(struct thread *); void vfp_restore_state(void); void vfp_save_state(struct thread *, struct pcb *); + +struct fpu_kern_ctx; + +/* + * Flags for fpu_kern_alloc_ctx(), fpu_kern_enter() and fpu_kern_thread(). + */ +#define FPU_KERN_NORMAL 0x0000 +#define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 + +struct fpu_kern_ctx *fpu_kern_alloc_ctx(u_int); +void fpu_kern_free_ctx(struct fpu_kern_ctx *); +int fpu_kern_enter(struct thread *, struct fpu_kern_ctx *, u_int); +int fpu_kern_leave(struct thread *, struct fpu_kern_ctx *); +int fpu_kern_thread(u_int); +int is_fpu_kern_thread(u_int); + #endif #endif Modified: head/sys/opencrypto/crypto.c ============================================================================== --- head/sys/opencrypto/crypto.c Thu Oct 20 09:06:39 2016 (r307666) +++ head/sys/opencrypto/crypto.c Thu Oct 20 09:22:10 2016 (r307667) @@ -81,7 +81,7 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include "cryptodev_if.h" -#if defined(__i386__) || defined(__amd64__) +#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) #include <machine/pcb.h> #endif @@ -1246,7 +1246,7 @@ crypto_proc(void) u_int32_t hid; int result, hint; -#if defined(__i386__) || defined(__amd64__) +#if defined(__i386__) || defined(__amd64__) || defined(__aarch64__) fpu_kern_thread(FPU_KERN_NORMAL); #endif _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
