FPU registers contain guest data and must be protected from information leak vulnerabilities in the kernel.
FPU register state for vCPUs are allocated from the globally-visible kernel heap. Change this to use process-local memory instead and thus prevent access (or prefetching) in any other context in the kernel. Signed-off-by: Julian Stecklina <jstec...@amazon.de> --- arch/x86/include/asm/kvm_host.h | 10 +++++++- arch/x86/kvm/x86.c | 42 ++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 55e51ff7e421..5dd29bfef77f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -36,6 +36,7 @@ #include <asm/asm.h> #include <asm/kvm_page_track.h> #include <asm/hyperv-tlfs.h> +#include <asm/proclocal.h> #define KVM_MAX_VCPUS 288 #define KVM_SOFT_MAX_VCPUS 240 @@ -530,7 +531,13 @@ struct kvm_vcpu_hv { cpumask_t tlb_flush; }; +struct kvm_vcpu_arch_hidden { + struct fpu guest_fpu; +}; + struct kvm_vcpu_arch { + struct proclocal hidden; + /* * rip and regs accesses must go through * kvm_{register,rip}_{read,write} functions. @@ -611,7 +618,6 @@ struct kvm_vcpu_arch { * host PRKU bits. */ struct fpu user_fpu; - struct fpu guest_fpu; u64 xcr0; u64 guest_supported_xcr0; @@ -1580,4 +1586,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu) #define put_smstate(type, buf, offset, val) \ *(type *)((buf) + (offset) - 0x7e00) = val +struct kvm_vcpu_arch_hidden *kvm_arch_vcpu_hidden_get(struct kvm_vcpu *vcpu); + #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 66d66d77caee..941fa3209607 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -37,6 +37,7 @@ #include <linux/vmalloc.h> #include <linux/export.h> #include <linux/moduleparam.h> +#include <linux/mm.h> #include <linux/mman.h> #include <linux/highmem.h> #include <linux/iommu.h> @@ -69,6 +70,7 @@ #include <asm/irq_remapping.h> #include <asm/mshyperv.h> #include <asm/hypervisor.h> +#include <asm/proclocal.h> #define CREATE_TRACE_POINTS #include "trace.h" @@ -3630,7 +3632,7 @@ static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) { - struct xregs_state *xsave = &vcpu->arch.guest_fpu.state.xsave; + struct xregs_state *xsave = &kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.xsave; u64 xstate_bv = xsave->header.xfeatures; u64 valid; @@ -3672,7 +3674,7 @@ static void fill_xsave(u8 *dest, struct kvm_vcpu *vcpu) static void load_xsave(struct kvm_vcpu *vcpu, u8 *src) { - struct xregs_state *xsave = &vcpu->arch.guest_fpu.state.xsave; + struct xregs_state *xsave = &kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.xsave; u64 xstate_bv = *(u64 *)(src + XSAVE_HDR_OFFSET); u64 valid; @@ -3720,7 +3722,7 @@ static void kvm_vcpu_ioctl_x86_get_xsave(struct kvm_vcpu *vcpu, fill_xsave((u8 *) guest_xsave->region, vcpu); } else { memcpy(guest_xsave->region, - &vcpu->arch.guest_fpu.state.fxsave, + &kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.fxsave, sizeof(struct fxregs_state)); *(u64 *)&guest_xsave->region[XSAVE_HDR_OFFSET / sizeof(u32)] = XFEATURE_MASK_FPSSE; @@ -3750,7 +3752,7 @@ static int kvm_vcpu_ioctl_x86_set_xsave(struct kvm_vcpu *vcpu, if (xstate_bv & ~XFEATURE_MASK_FPSSE || mxcsr & ~mxcsr_feature_mask) return -EINVAL; - memcpy(&vcpu->arch.guest_fpu.state.fxsave, + memcpy(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.fxsave, guest_xsave->region, sizeof(struct fxregs_state)); } return 0; @@ -7996,7 +7998,7 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) preempt_disable(); copy_fpregs_to_fpstate(&vcpu->arch.user_fpu); /* PKRU is separately restored in kvm_x86_ops->run. */ - __copy_kernel_to_fpregs(&vcpu->arch.guest_fpu.state, + __copy_kernel_to_fpregs(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state, ~XFEATURE_MASK_PKRU); preempt_enable(); trace_kvm_fpu(1); @@ -8006,7 +8008,7 @@ static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) { preempt_disable(); - copy_fpregs_to_fpstate(&vcpu->arch.guest_fpu); + copy_fpregs_to_fpstate(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu); copy_kernel_to_fpregs(&vcpu->arch.user_fpu.state); preempt_enable(); ++vcpu->stat.fpu_reload; @@ -8501,7 +8503,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) vcpu_load(vcpu); - fxsave = &vcpu->arch.guest_fpu.state.fxsave; + fxsave = &kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.fxsave; memcpy(fpu->fpr, fxsave->st_space, 128); fpu->fcw = fxsave->cwd; fpu->fsw = fxsave->swd; @@ -8521,8 +8523,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) vcpu_load(vcpu); - fxsave = &vcpu->arch.guest_fpu.state.fxsave; - + fxsave = &kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.fxsave; memcpy(fxsave->st_space, fpu->fpr, 128); fxsave->cwd = fpu->fcw; fxsave->swd = fpu->fsw; @@ -8577,9 +8578,9 @@ static int sync_regs(struct kvm_vcpu *vcpu) static void fx_init(struct kvm_vcpu *vcpu) { - fpstate_init(&vcpu->arch.guest_fpu.state); + fpstate_init(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state); if (boot_cpu_has(X86_FEATURE_XSAVES)) - vcpu->arch.guest_fpu.state.xsave.header.xcomp_bv = + kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.xsave.header.xcomp_bv = host_xcr0 | XSTATE_COMPACTION_ENABLED; /* @@ -8703,11 +8704,11 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) */ if (init_event) kvm_put_guest_fpu(vcpu); - mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu.state.xsave, + mpx_state_buffer = get_xsave_addr(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.xsave, XFEATURE_MASK_BNDREGS); if (mpx_state_buffer) memset(mpx_state_buffer, 0, sizeof(struct mpx_bndreg_state)); - mpx_state_buffer = get_xsave_addr(&vcpu->arch.guest_fpu.state.xsave, + mpx_state_buffer = get_xsave_addr(&kvm_arch_vcpu_hidden_get(vcpu)->guest_fpu.state.xsave, XFEATURE_MASK_BNDCSR); if (mpx_state_buffer) memset(mpx_state_buffer, 0, sizeof(struct mpx_bndcsr)); @@ -8892,11 +8893,21 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu) struct static_key kvm_no_apic_vcpu __read_mostly; EXPORT_SYMBOL_GPL(kvm_no_apic_vcpu); +struct kvm_vcpu_arch_hidden *kvm_arch_vcpu_hidden_get(struct kvm_vcpu *vcpu) +{ + return proclocal_get(&vcpu->arch.hidden, struct kvm_vcpu_arch_hidden); +} +EXPORT_SYMBOL_GPL(kvm_arch_vcpu_hidden_get); + int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) { struct page *page; int r; + r = kalloc_proclocal(&vcpu->arch.hidden, sizeof(struct kvm_vcpu_arch_hidden)); + if (r) + goto fail; + vcpu->arch.apicv_active = kvm_x86_ops->get_enable_apicv(vcpu); vcpu->arch.emulate_ctxt.ops = &emulate_ops; if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) @@ -8907,7 +8918,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) { r = -ENOMEM; - goto fail; + goto fail_free_hidden; } vcpu->arch.pio_data = page_address(page); @@ -8963,6 +8974,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) kvm_mmu_destroy(vcpu); fail_free_pio_data: free_page((unsigned long)vcpu->arch.pio_data); +fail_free_hidden: + kfree_proclocal(&vcpu->arch.hidden); fail: return r; } @@ -8981,6 +8994,7 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) free_page((unsigned long)vcpu->arch.pio_data); if (!lapic_in_kernel(vcpu)) static_key_slow_dec(&kvm_no_apic_vcpu); + kfree_proclocal(&vcpu->arch.hidden); } void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) -- 2.17.1