Author: markj Date: Mon Oct 19 15:24:35 2020 New Revision: 366834 URL: https://svnweb.freebsd.org/changeset/base/366834
Log: vmx: Implement pmap (de)activation in C Rewrite the code that maintains pm_active and invalidates EPTP-tagged TLB entries in C. Previously this work was done in vmx_enter_guest(), in assembly, but there is no good reason for that and it makes the TLB invalidation algorithm for nested page tables harder to review. No functional change intended. Now, an error from the invept instruction results in a kernel panic rather than a vmexit. Such errors should occur only as a result of VMM bugs. Reviewed by: grehan, kib MFC after: 2 weeks Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D26830 Modified: head/sys/amd64/vmm/intel/vmx.c head/sys/amd64/vmm/intel/vmx.h head/sys/amd64/vmm/intel/vmx_genassym.c head/sys/amd64/vmm/intel/vmx_support.S Modified: head/sys/amd64/vmm/intel/vmx.c ============================================================================== --- head/sys/amd64/vmm/intel/vmx.c Mon Oct 19 13:10:21 2020 (r366833) +++ head/sys/amd64/vmm/intel/vmx.c Mon Oct 19 15:24:35 2020 (r366834) @@ -2835,7 +2835,6 @@ vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, str switch (rc) { case VMX_VMRESUME_ERROR: case VMX_VMLAUNCH_ERROR: - case VMX_INVEPT_ERROR: vmexit->u.vmx.inst_type = rc; break; default: @@ -2940,6 +2939,29 @@ vmx_dr_leave_guest(struct vmxctx *vmxctx) write_rflags(read_rflags() | vmxctx->host_tf); } +static __inline void +vmx_pmap_activate(struct vmx *vmx, pmap_t pmap) +{ + long eptgen; + int cpu; + + cpu = curcpu; + + CPU_SET_ATOMIC(cpu, &pmap->pm_active); + eptgen = atomic_load_long(&pmap->pm_eptgen); + if (eptgen != vmx->eptgen[cpu]) { + vmx->eptgen[cpu] = eptgen; + invept(INVEPT_TYPE_SINGLE_CONTEXT, + (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 }); + } +} + +static __inline void +vmx_pmap_deactivate(struct vmx *vmx, pmap_t pmap) +{ + CPU_CLR_ATOMIC(curcpu, &pmap->pm_active); +} + static int vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, struct vm_eventinfo *evinfo) @@ -3088,11 +3110,19 @@ vmx_run(void *arg, int vcpu, register_t rip, pmap_t pm */ vmx_msr_guest_enter_tsc_aux(vmx, vcpu); - vmx_run_trace(vmx, vcpu); vmx_dr_enter_guest(vmxctx); + + /* + * Mark the EPT as active on this host CPU and invalidate + * EPTP-tagged TLB entries if required. + */ + vmx_pmap_activate(vmx, pmap); + + vmx_run_trace(vmx, vcpu); rc = vmx_enter_guest(vmxctx, vmx, launched); - vmx_dr_leave_guest(vmxctx); + vmx_pmap_deactivate(vmx, pmap); + vmx_dr_leave_guest(vmxctx); vmx_msr_guest_exit_tsc_aux(vmx, vcpu); bare_lgdt(&gdtr); Modified: head/sys/amd64/vmm/intel/vmx.h ============================================================================== --- head/sys/amd64/vmm/intel/vmx.h Mon Oct 19 13:10:21 2020 (r366833) +++ head/sys/amd64/vmm/intel/vmx.h Mon Oct 19 15:24:35 2020 (r366834) @@ -142,7 +142,6 @@ CTASSERT((offsetof(struct vmx, pir_desc[0]) & 63) == 0 #define VMX_GUEST_VMEXIT 0 #define VMX_VMRESUME_ERROR 1 #define VMX_VMLAUNCH_ERROR 2 -#define VMX_INVEPT_ERROR 3 int vmx_enter_guest(struct vmxctx *ctx, struct vmx *vmx, int launched); void vmx_call_isr(uintptr_t entry); Modified: head/sys/amd64/vmm/intel/vmx_genassym.c ============================================================================== --- head/sys/amd64/vmm/intel/vmx_genassym.c Mon Oct 19 13:10:21 2020 (r366833) +++ head/sys/amd64/vmm/intel/vmx_genassym.c Mon Oct 19 15:24:35 2020 (r366834) @@ -70,17 +70,12 @@ ASSYM(VMXCTX_HOST_RSP, offsetof(struct vmxctx, host_rs ASSYM(VMXCTX_HOST_RBX, offsetof(struct vmxctx, host_rbx)); ASSYM(VMXCTX_INST_FAIL_STATUS, offsetof(struct vmxctx, inst_fail_status)); -ASSYM(VMXCTX_PMAP, offsetof(struct vmxctx, pmap)); -ASSYM(VMX_EPTGEN, offsetof(struct vmx, eptgen)); -ASSYM(VMX_EPTP, offsetof(struct vmx, eptp)); - ASSYM(VM_FAIL_INVALID, VM_FAIL_INVALID); ASSYM(VM_FAIL_VALID, VM_FAIL_VALID); ASSYM(VMX_GUEST_VMEXIT, VMX_GUEST_VMEXIT); ASSYM(VMX_VMRESUME_ERROR, VMX_VMRESUME_ERROR); ASSYM(VMX_VMLAUNCH_ERROR, VMX_VMLAUNCH_ERROR); -ASSYM(VMX_INVEPT_ERROR, VMX_INVEPT_ERROR); ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); Modified: head/sys/amd64/vmm/intel/vmx_support.S ============================================================================== --- head/sys/amd64/vmm/intel/vmx_support.S Mon Oct 19 13:10:21 2020 (r366833) +++ head/sys/amd64/vmm/intel/vmx_support.S Mon Oct 19 15:24:35 2020 (r366834) @@ -148,33 +148,6 @@ ENTRY(vmx_enter_guest) */ VMX_HOST_SAVE - /* - * Activate guest pmap on this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %eax - LK btsl %eax, PM_ACTIVE(%r11) - - /* - * If 'vmx->eptgen[curcpu]' is not identical to 'pmap->pm_eptgen' - * then we must invalidate all mappings associated with this EPTP. - */ - movq PM_EPTGEN(%r11), %r10 - cmpq %r10, VMX_EPTGEN(%rsi, %rax, 8) - je guest_restore - - /* Refresh 'vmx->eptgen[curcpu]' */ - movq %r10, VMX_EPTGEN(%rsi, %rax, 8) - - /* Setup the invept descriptor on the host stack */ - mov %rsp, %r11 - movq VMX_EPTP(%rsi), %rax - movq %rax, -16(%r11) - movq $0x0, -8(%r11) - mov $0x1, %eax /* Single context invalidate */ - invept -16(%r11), %rax - jbe invept_error /* Check invept instruction error */ - guest_restore: movl %edx, %r8d cmpb $0, guest_l1d_flush_sw(%rip) @@ -208,10 +181,6 @@ do_launch: movl $VMX_VMLAUNCH_ERROR, %eax jmp decode_inst_error -invept_error: - movl $VMX_INVEPT_ERROR, %eax - jmp decode_inst_error - decode_inst_error: movl $VM_FAIL_VALID, %r11d jz inst_error @@ -224,13 +193,6 @@ inst_error: * it as a scratch register beyond this point. */ - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) - VMX_HOST_RESTORE VLEAVE ret @@ -248,13 +210,6 @@ vmx_exit_guest_flush_rsb: */ VMX_GUEST_SAVE - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) - VMX_HOST_RESTORE VMX_GUEST_CLOBBER @@ -290,13 +245,6 @@ vmx_exit_guest: * Save guest state that is not automatically saved in the vmcs. */ VMX_GUEST_SAVE - - /* - * Deactivate guest pmap from this cpu. - */ - movq VMXCTX_PMAP(%rdi), %r11 - movl PCPU(CPUID), %r10d - LK btrl %r10d, PM_ACTIVE(%r11) VMX_HOST_RESTORE _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
