Module Name: src
Committed By: maxv
Date: Sat Feb 23 08:19:16 UTC 2019
Modified Files:
src/sys/dev/nvmm/x86: nvmm_x86_svm.c nvmm_x86_vmx.c
Log Message:
Reorder the functions, and constify setstate. No functional change.
To generate a diff of this commit:
cvs rdiff -u -r1.29 -r1.30 src/sys/dev/nvmm/x86/nvmm_x86_svm.c
cvs rdiff -u -r1.11 -r1.12 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/nvmm/x86/nvmm_x86_svm.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.29 src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.30
--- src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.29 Thu Feb 21 12:17:52 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_svm.c Sat Feb 23 08:19:16 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $ */
+/* $NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.29 2019/02/21 12:17:52 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.30 2019/02/23 08:19:16 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1468,265 +1468,7 @@ svm_vcpu_msr_allow(uint8_t *bitmap, uint
}
}
-static void
-svm_asid_alloc(struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
- size_t i, oct, bit;
-
- mutex_enter(&svm_asidlock);
-
- for (i = 0; i < svm_maxasid; i++) {
- oct = i / 8;
- bit = i % 8;
-
- if (svm_asidmap[oct] & __BIT(bit)) {
- continue;
- }
-
- svm_asidmap[oct] |= __BIT(bit);
- vmcb->ctrl.guest_asid = i;
- mutex_exit(&svm_asidlock);
- return;
- }
-
- /*
- * No free ASID. Use the last one, which is shared and requires
- * special TLB handling.
- */
- cpudata->shared_asid = true;
- vmcb->ctrl.guest_asid = svm_maxasid - 1;
- mutex_exit(&svm_asidlock);
-}
-static void
-svm_asid_free(struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
- size_t oct, bit;
-
- if (cpudata->shared_asid) {
- return;
- }
-
- oct = vmcb->ctrl.guest_asid / 8;
- bit = vmcb->ctrl.guest_asid % 8;
-
- mutex_enter(&svm_asidlock);
- svm_asidmap[oct] &= ~__BIT(bit);
- mutex_exit(&svm_asidlock);
-}
-
-static void
-svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
- struct vmcb *vmcb = cpudata->vmcb;
-
- /* Allow reads/writes of Control Registers. */
- vmcb->ctrl.intercept_cr = 0;
-
- /* Allow reads/writes of Debug Registers. */
- vmcb->ctrl.intercept_dr = 0;
-
- /* Allow exceptions 0 to 31. */
- vmcb->ctrl.intercept_vec = 0;
-
- /*
- * Allow:
- * - SMI [smm interrupts]
- * - VINTR [virtual interrupts]
- * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP]
- * - RIDTR [reads of IDTR]
- * - RGDTR [reads of GDTR]
- * - RLDTR [reads of LDTR]
- * - RTR [reads of TR]
- * - WIDTR [writes of IDTR]
- * - WGDTR [writes of GDTR]
- * - WLDTR [writes of LDTR]
- * - WTR [writes of TR]
- * - RDTSC [rdtsc instruction]
- * - PUSHF [pushf instruction]
- * - POPF [popf instruction]
- * - IRET [iret instruction]
- * - INTN [int $n instructions]
- * - INVD [invd instruction]
- * - PAUSE [pause instruction]
- * - INVLPG [invplg instruction]
- * - TASKSW [task switches]
- *
- * Intercept the rest below.
- */
- vmcb->ctrl.intercept_misc1 =
- VMCB_CTRL_INTERCEPT_INTR |
- VMCB_CTRL_INTERCEPT_NMI |
- VMCB_CTRL_INTERCEPT_INIT |
- VMCB_CTRL_INTERCEPT_RDPMC |
- VMCB_CTRL_INTERCEPT_CPUID |
- VMCB_CTRL_INTERCEPT_RSM |
- VMCB_CTRL_INTERCEPT_HLT |
- VMCB_CTRL_INTERCEPT_INVLPGA |
- VMCB_CTRL_INTERCEPT_IOIO_PROT |
- VMCB_CTRL_INTERCEPT_MSR_PROT |
- VMCB_CTRL_INTERCEPT_FERR_FREEZE |
- VMCB_CTRL_INTERCEPT_SHUTDOWN;
-
- /*
- * Allow:
- * - ICEBP [icebp instruction]
- * - WBINVD [wbinvd instruction]
- * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction]
- *
- * Intercept the rest below.
- */
- vmcb->ctrl.intercept_misc2 =
- VMCB_CTRL_INTERCEPT_VMRUN |
- VMCB_CTRL_INTERCEPT_VMMCALL |
- VMCB_CTRL_INTERCEPT_VMLOAD |
- VMCB_CTRL_INTERCEPT_VMSAVE |
- VMCB_CTRL_INTERCEPT_STGI |
- VMCB_CTRL_INTERCEPT_CLGI |
- VMCB_CTRL_INTERCEPT_SKINIT |
- VMCB_CTRL_INTERCEPT_RDTSCP |
- VMCB_CTRL_INTERCEPT_MONITOR |
- VMCB_CTRL_INTERCEPT_MWAIT |
- VMCB_CTRL_INTERCEPT_XSETBV;
-
- /* Intercept all I/O accesses. */
- memset(cpudata->iobm, 0xFF, IOBM_SIZE);
- vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
-
- /*
- * Allow:
- * - EFER [read]
- * - STAR [read, write]
- * - LSTAR [read, write]
- * - CSTAR [read, write]
- * - SFMASK [read, write]
- * - KERNELGSBASE [read, write]
- * - SYSENTER_CS [read, write]
- * - SYSENTER_ESP [read, write]
- * - SYSENTER_EIP [read, write]
- * - FSBASE [read, write]
- * - GSBASE [read, write]
- * - PAT [read, write]
- * - TSC [read]
- *
- * Intercept the rest.
- */
- memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
- svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
- vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
-
- /* Generate ASID. */
- svm_asid_alloc(vcpu);
-
- /* Virtual TPR. */
- vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING;
-
- /* Enable Nested Paging. */
- vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP;
- vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0];
-
- /* Must always be set. */
- vmcb->state.efer = EFER_SVME;
- cpudata->gxcr0 = XCR0_X87;
-
- /* Init XSAVE header. */
- cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
- cpudata->gfpu.xsh_xcomp_bv = 0;
-
- /* Set guest TSC to zero, more or less. */
- cpudata->tsc_offset = -cpu_counter();
-
- /* These MSRs are static. */
- cpudata->star = rdmsr(MSR_STAR);
- cpudata->lstar = rdmsr(MSR_LSTAR);
- cpudata->cstar = rdmsr(MSR_CSTAR);
- cpudata->sfmask = rdmsr(MSR_SFMASK);
-}
-
-static int
-svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata;
- int error;
-
- /* Allocate the SVM cpudata. */
- cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
- roundup(sizeof(*cpudata), PAGE_SIZE), 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- vcpu->cpudata = cpudata;
-
- /* VMCB */
- error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
- VMCB_NPAGES);
- if (error)
- goto error;
-
- /* I/O Bitmap */
- error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
- IOBM_NPAGES);
- if (error)
- goto error;
-
- /* MSR Bitmap */
- error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
- MSRBM_NPAGES);
- if (error)
- goto error;
-
- /* Init the VCPU info. */
- svm_vcpu_init(mach, vcpu);
-
- return 0;
-
-error:
- if (cpudata->vmcb_pa) {
- svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
- VMCB_NPAGES);
- }
- if (cpudata->iobm_pa) {
- svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
- IOBM_NPAGES);
- }
- if (cpudata->msrbm_pa) {
- svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
- MSRBM_NPAGES);
- }
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
- return error;
-}
-
-static void
-svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct svm_cpudata *cpudata = vcpu->cpudata;
-
- svm_asid_free(vcpu);
-
- svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
- svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
- svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
-
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
-}
#define SVM_SEG_ATTRIB_TYPE __BITS(4,0)
#define SVM_SEG_ATTRIB_DPL __BITS(6,5)
@@ -1737,7 +1479,8 @@ svm_vcpu_destroy(struct nvmm_machine *ma
#define SVM_SEG_ATTRIB_GRAN __BIT(11)
static void
-svm_vcpu_setstate_seg(struct nvmm_x64_state_seg *seg, struct vmcb_segment *vseg)
+svm_vcpu_setstate_seg(const struct nvmm_x64_state_seg *seg,
+ struct vmcb_segment *vseg)
{
vseg->selector = seg->selector;
vseg->attrib =
@@ -1768,7 +1511,7 @@ svm_vcpu_getstate_seg(struct nvmm_x64_st
}
static inline bool
-svm_state_tlb_flush(struct vmcb *vmcb, struct nvmm_x64_state *state,
+svm_state_tlb_flush(const struct vmcb *vmcb, const struct nvmm_x64_state *state,
uint64_t flags)
{
if (flags & NVMM_X64_STATE_CRS) {
@@ -1798,7 +1541,7 @@ svm_state_tlb_flush(struct vmcb *vmcb, s
static void
svm_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
{
- struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
+ const struct nvmm_x64_state *state = data;
struct svm_cpudata *cpudata = vcpu->cpudata;
struct vmcb *vmcb = cpudata->vmcb;
struct fxsave *fpustate;
@@ -1869,9 +1612,10 @@ svm_vcpu_setstate(struct nvmm_cpu *vcpu,
}
if (flags & NVMM_X64_STATE_MSRS) {
- /* Bit EFER_SVME is mandatory. */
+ /*
+ * EFER_SVME is mandatory.
+ */
vmcb->state.efer = state->msrs[NVMM_X64_MSR_EFER] | EFER_SVME;
-
vmcb->state.star = state->msrs[NVMM_X64_MSR_STAR];
vmcb->state.lstar = state->msrs[NVMM_X64_MSR_LSTAR];
vmcb->state.cstar = state->msrs[NVMM_X64_MSR_CSTAR];
@@ -2024,6 +1768,251 @@ svm_vcpu_getstate(struct nvmm_cpu *vcpu,
/* -------------------------------------------------------------------------- */
static void
+svm_asid_alloc(struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+ size_t i, oct, bit;
+
+ mutex_enter(&svm_asidlock);
+
+ for (i = 0; i < svm_maxasid; i++) {
+ oct = i / 8;
+ bit = i % 8;
+
+ if (svm_asidmap[oct] & __BIT(bit)) {
+ continue;
+ }
+
+ svm_asidmap[oct] |= __BIT(bit);
+ vmcb->ctrl.guest_asid = i;
+ mutex_exit(&svm_asidlock);
+ return;
+ }
+
+ /*
+ * No free ASID. Use the last one, which is shared and requires
+ * special TLB handling.
+ */
+ cpudata->shared_asid = true;
+ vmcb->ctrl.guest_asid = svm_maxasid - 1;
+ mutex_exit(&svm_asidlock);
+}
+
+static void
+svm_asid_free(struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+ size_t oct, bit;
+
+ if (cpudata->shared_asid) {
+ return;
+ }
+
+ oct = vmcb->ctrl.guest_asid / 8;
+ bit = vmcb->ctrl.guest_asid % 8;
+
+ mutex_enter(&svm_asidlock);
+ svm_asidmap[oct] &= ~__BIT(bit);
+ mutex_exit(&svm_asidlock);
+}
+
+static void
+svm_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+ struct vmcb *vmcb = cpudata->vmcb;
+
+ /* Allow reads/writes of Control Registers. */
+ vmcb->ctrl.intercept_cr = 0;
+
+ /* Allow reads/writes of Debug Registers. */
+ vmcb->ctrl.intercept_dr = 0;
+
+ /* Allow exceptions 0 to 31. */
+ vmcb->ctrl.intercept_vec = 0;
+
+ /*
+ * Allow:
+ * - SMI [smm interrupts]
+ * - VINTR [virtual interrupts]
+ * - CR0_SPEC [CR0 writes changing other fields than CR0.TS or CR0.MP]
+ * - RIDTR [reads of IDTR]
+ * - RGDTR [reads of GDTR]
+ * - RLDTR [reads of LDTR]
+ * - RTR [reads of TR]
+ * - WIDTR [writes of IDTR]
+ * - WGDTR [writes of GDTR]
+ * - WLDTR [writes of LDTR]
+ * - WTR [writes of TR]
+ * - RDTSC [rdtsc instruction]
+ * - PUSHF [pushf instruction]
+ * - POPF [popf instruction]
+ * - IRET [iret instruction]
+ * - INTN [int $n instructions]
+ * - INVD [invd instruction]
+ * - PAUSE [pause instruction]
+ * - INVLPG [invplg instruction]
+ * - TASKSW [task switches]
+ *
+ * Intercept the rest below.
+ */
+ vmcb->ctrl.intercept_misc1 =
+ VMCB_CTRL_INTERCEPT_INTR |
+ VMCB_CTRL_INTERCEPT_NMI |
+ VMCB_CTRL_INTERCEPT_INIT |
+ VMCB_CTRL_INTERCEPT_RDPMC |
+ VMCB_CTRL_INTERCEPT_CPUID |
+ VMCB_CTRL_INTERCEPT_RSM |
+ VMCB_CTRL_INTERCEPT_HLT |
+ VMCB_CTRL_INTERCEPT_INVLPGA |
+ VMCB_CTRL_INTERCEPT_IOIO_PROT |
+ VMCB_CTRL_INTERCEPT_MSR_PROT |
+ VMCB_CTRL_INTERCEPT_FERR_FREEZE |
+ VMCB_CTRL_INTERCEPT_SHUTDOWN;
+
+ /*
+ * Allow:
+ * - ICEBP [icebp instruction]
+ * - WBINVD [wbinvd instruction]
+ * - WCR_SPEC(0..15) [writes of CR0-15, received after instruction]
+ *
+ * Intercept the rest below.
+ */
+ vmcb->ctrl.intercept_misc2 =
+ VMCB_CTRL_INTERCEPT_VMRUN |
+ VMCB_CTRL_INTERCEPT_VMMCALL |
+ VMCB_CTRL_INTERCEPT_VMLOAD |
+ VMCB_CTRL_INTERCEPT_VMSAVE |
+ VMCB_CTRL_INTERCEPT_STGI |
+ VMCB_CTRL_INTERCEPT_CLGI |
+ VMCB_CTRL_INTERCEPT_SKINIT |
+ VMCB_CTRL_INTERCEPT_RDTSCP |
+ VMCB_CTRL_INTERCEPT_MONITOR |
+ VMCB_CTRL_INTERCEPT_MWAIT |
+ VMCB_CTRL_INTERCEPT_XSETBV;
+
+ /* Intercept all I/O accesses. */
+ memset(cpudata->iobm, 0xFF, IOBM_SIZE);
+ vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
+
+ /* Allow direct access to certain MSRs. */
+ memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_CR_PAT, true, true);
+ svm_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
+ vmcb->ctrl.msrpm_base_pa = cpudata->msrbm_pa;
+
+ /* Generate ASID. */
+ svm_asid_alloc(vcpu);
+
+ /* Virtual TPR. */
+ vmcb->ctrl.v = VMCB_CTRL_V_INTR_MASKING;
+
+ /* Enable Nested Paging. */
+ vmcb->ctrl.enable1 = VMCB_CTRL_ENABLE_NP;
+ vmcb->ctrl.n_cr3 = mach->vm->vm_map.pmap->pm_pdirpa[0];
+
+ /* Must always be set. */
+ vmcb->state.efer = EFER_SVME;
+ cpudata->gxcr0 = XCR0_X87;
+
+ /* Init XSAVE header. */
+ cpudata->gfpu.xsh_xstate_bv = svm_xcr0_mask;
+ cpudata->gfpu.xsh_xcomp_bv = 0;
+
+ /* Set guest TSC to zero, more or less. */
+ cpudata->tsc_offset = -cpu_counter();
+
+ /* These MSRs are static. */
+ cpudata->star = rdmsr(MSR_STAR);
+ cpudata->lstar = rdmsr(MSR_LSTAR);
+ cpudata->cstar = rdmsr(MSR_CSTAR);
+ cpudata->sfmask = rdmsr(MSR_SFMASK);
+}
+
+static int
+svm_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata;
+ int error;
+
+ /* Allocate the SVM cpudata. */
+ cpudata = (struct svm_cpudata *)uvm_km_alloc(kernel_map,
+ roundup(sizeof(*cpudata), PAGE_SIZE), 0,
+ UVM_KMF_WIRED|UVM_KMF_ZERO);
+ vcpu->cpudata = cpudata;
+
+ /* VMCB */
+ error = svm_memalloc(&cpudata->vmcb_pa, (vaddr_t *)&cpudata->vmcb,
+ VMCB_NPAGES);
+ if (error)
+ goto error;
+
+ /* I/O Bitmap */
+ error = svm_memalloc(&cpudata->iobm_pa, (vaddr_t *)&cpudata->iobm,
+ IOBM_NPAGES);
+ if (error)
+ goto error;
+
+ /* MSR Bitmap */
+ error = svm_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
+ MSRBM_NPAGES);
+ if (error)
+ goto error;
+
+ /* Init the VCPU info. */
+ svm_vcpu_init(mach, vcpu);
+
+ return 0;
+
+error:
+ if (cpudata->vmcb_pa) {
+ svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb,
+ VMCB_NPAGES);
+ }
+ if (cpudata->iobm_pa) {
+ svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm,
+ IOBM_NPAGES);
+ }
+ if (cpudata->msrbm_pa) {
+ svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
+ MSRBM_NPAGES);
+ }
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
+ return error;
+}
+
+static void
+svm_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct svm_cpudata *cpudata = vcpu->cpudata;
+
+ svm_asid_free(vcpu);
+
+ svm_memfree(cpudata->vmcb_pa, (vaddr_t)cpudata->vmcb, VMCB_NPAGES);
+ svm_memfree(cpudata->iobm_pa, (vaddr_t)cpudata->iobm, IOBM_NPAGES);
+ svm_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
+
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
+}
+
+/* -------------------------------------------------------------------------- */
+
+static void
svm_tlb_flush(struct pmap *pm)
{
struct nvmm_machine *mach = pm->pm_data;
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.11 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.12
--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.11 Fri Feb 22 12:24:34 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Feb 23 08:19:16 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.11 2019/02/22 12:24:34 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.12 2019/02/23 08:19:16 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -1911,66 +1911,6 @@ vmx_memfree(paddr_t pa, vaddr_t va, size
/* -------------------------------------------------------------------------- */
static void
-vmx_asid_alloc(struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- size_t i, oct, bit;
-
- mutex_enter(&vmx_asidlock);
-
- for (i = 0; i < vmx_maxasid; i++) {
- oct = i / 8;
- bit = i % 8;
-
- if (vmx_asidmap[oct] & __BIT(bit)) {
- continue;
- }
-
- cpudata->asid = i;
-
- vmx_asidmap[oct] |= __BIT(bit);
- vmx_vmwrite(VMCS_VPID, i);
- mutex_exit(&vmx_asidlock);
- return;
- }
-
- mutex_exit(&vmx_asidlock);
-
- panic("%s: impossible", __func__);
-}
-
-static void
-vmx_asid_free(struct nvmm_cpu *vcpu)
-{
- size_t oct, bit;
- uint64_t asid;
-
- vmx_vmread(VMCS_VPID, &asid);
-
- oct = asid / 8;
- bit = asid % 8;
-
- mutex_enter(&vmx_asidlock);
- vmx_asidmap[oct] &= ~__BIT(bit);
- mutex_exit(&vmx_asidlock);
-}
-
-static void
-vmx_init_asid(uint32_t maxasid)
-{
- size_t allocsz;
-
- mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE);
-
- vmx_maxasid = maxasid;
- allocsz = roundup(maxasid, 8) / 8;
- vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP);
-
- /* ASID 0 is reserved for the host. */
- vmx_asidmap[0] |= __BIT(0);
-}
-
-static void
vmx_vcpu_msr_allow(uint8_t *bitmap, uint64_t msr, bool read, bool write)
{
uint64_t byte;
@@ -1996,518 +1936,566 @@ vmx_vcpu_msr_allow(uint8_t *bitmap, uint
}
}
+#define VMX_SEG_ATTRIB_TYPE __BITS(4,0)
+#define VMX_SEG_ATTRIB_DPL __BITS(6,5)
+#define VMX_SEG_ATTRIB_P __BIT(7)
+#define VMX_SEG_ATTRIB_AVL __BIT(12)
+#define VMX_SEG_ATTRIB_LONG __BIT(13)
+#define VMX_SEG_ATTRIB_DEF32 __BIT(14)
+#define VMX_SEG_ATTRIB_GRAN __BIT(15)
+#define VMX_SEG_ATTRIB_UNUSABLE __BIT(16)
+
static void
-vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+vmx_vcpu_setstate_seg(const struct nvmm_x64_state_seg *segs, int idx)
{
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- struct vmcs *vmcs = cpudata->vmcs;
- struct msr_entry *gmsr = cpudata->gmsr;
- extern uint8_t vmx_resume_rip;
- uint64_t rev, eptp;
+ uint64_t attrib;
- rev = vmx_get_revision();
+ attrib =
+ __SHIFTIN(segs[idx].attrib.type, VMX_SEG_ATTRIB_TYPE) |
+ __SHIFTIN(segs[idx].attrib.dpl, VMX_SEG_ATTRIB_DPL) |
+ __SHIFTIN(segs[idx].attrib.p, VMX_SEG_ATTRIB_P) |
+ __SHIFTIN(segs[idx].attrib.avl, VMX_SEG_ATTRIB_AVL) |
+ __SHIFTIN(segs[idx].attrib.lng, VMX_SEG_ATTRIB_LONG) |
+ __SHIFTIN(segs[idx].attrib.def32, VMX_SEG_ATTRIB_DEF32) |
+ __SHIFTIN(segs[idx].attrib.gran, VMX_SEG_ATTRIB_GRAN) |
+ (!segs[idx].attrib.p ? VMX_SEG_ATTRIB_UNUSABLE : 0);
- memset(vmcs, 0, VMCS_SIZE);
- vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION);
- vmcs->abort = 0;
+ if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) {
+ vmx_vmwrite(vmx_guest_segs[idx].selector, segs[idx].selector);
+ vmx_vmwrite(vmx_guest_segs[idx].attrib, attrib);
+ }
+ vmx_vmwrite(vmx_guest_segs[idx].limit, segs[idx].limit);
+ vmx_vmwrite(vmx_guest_segs[idx].base, segs[idx].base);
+}
- vmx_vmcs_enter(vcpu);
+static void
+vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx)
+{
+ uint64_t attrib = 0;
- /* No link pointer. */
- vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF);
+ if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) {
+ vmx_vmread(vmx_guest_segs[idx].selector, &segs[idx].selector);
+ vmx_vmread(vmx_guest_segs[idx].attrib, &attrib);
+ }
+ vmx_vmread(vmx_guest_segs[idx].limit, &segs[idx].limit);
+ vmx_vmread(vmx_guest_segs[idx].base, &segs[idx].base);
- /* Install the CTLSs. */
- vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls);
- vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls);
- vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2);
- vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls);
- vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls);
+ segs[idx].attrib.type = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_TYPE);
+ segs[idx].attrib.dpl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DPL);
+ segs[idx].attrib.p = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_P);
+ segs[idx].attrib.avl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_AVL);
+ segs[idx].attrib.lng = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_LONG);
+ segs[idx].attrib.def32 = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DEF32);
+ segs[idx].attrib.gran = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_GRAN);
+ if (attrib & VMX_SEG_ATTRIB_UNUSABLE) {
+ segs[idx].attrib.p = 0;
+ }
+}
- /* Allow direct access to certain MSRs. */
- memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
- vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES,
- true, false);
- vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa);
+static inline bool
+vmx_state_tlb_flush(const struct nvmm_x64_state *state, uint64_t flags)
+{
+ uint64_t cr0, cr3, cr4, efer;
- /*
- * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This
- * includes the L1D_FLUSH MSR, to mitigate L1TF.
- */
- gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR;
- gmsr[VMX_MSRLIST_STAR].val = 0;
- gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR;
- gmsr[VMX_MSRLIST_LSTAR].val = 0;
- gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR;
- gmsr[VMX_MSRLIST_CSTAR].val = 0;
- gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK;
- gmsr[VMX_MSRLIST_SFMASK].val = 0;
- gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE;
- gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0;
- gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD;
- gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH;
- vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa);
- vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa);
- vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr);
- vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR);
+ if (flags & NVMM_X64_STATE_CRS) {
+ vmx_vmread(VMCS_GUEST_CR0, &cr0);
+ if ((cr0 ^ state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) {
+ return true;
+ }
+ vmx_vmread(VMCS_GUEST_CR3, &cr3);
+ if (cr3 != state->crs[NVMM_X64_CR_CR3]) {
+ return true;
+ }
+ vmx_vmread(VMCS_GUEST_CR4, &cr4);
+ if ((cr4 ^ state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) {
+ return true;
+ }
+ }
- /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */
- vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET);
- vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET);
+ if (flags & NVMM_X64_STATE_MSRS) {
+ vmx_vmread(VMCS_GUEST_IA32_EFER, &efer);
+ if ((efer ^
+ state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) {
+ return true;
+ }
+ }
- /* Force CR4_VMXE to zero. */
- vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE);
+ return false;
+}
- /* Set the Host state for resuming. */
- vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip);
- vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
- vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0);
- vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0);
- vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0);
- vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt);
- vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT));
- vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER));
- vmx_vmwrite(VMCS_HOST_CR0, rcr0());
+static void
+vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
+{
+ const struct nvmm_x64_state *state = data;
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ struct fxsave *fpustate;
+ uint64_t ctls1, intstate;
- /* Generate ASID. */
- vmx_asid_alloc(vcpu);
+ vmx_vmcs_enter(vcpu);
- /* Enable Extended Paging, 4-Level. */
- eptp =
- __SHIFTIN(vmx_eptp_type, EPTP_TYPE) |
- __SHIFTIN(4-1, EPTP_WALKLEN) |
- EPTP_FLAGS_AD |
- mach->vm->vm_map.pmap->pm_pdirpa[0];
- vmx_vmwrite(VMCS_EPTP, eptp);
+ if (vmx_state_tlb_flush(state, flags)) {
+ cpudata->gtlb_want_flush = true;
+ }
- /* Init IA32_MISC_ENABLE. */
- cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE);
- cpudata->gmsr_misc_enable &=
- ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN);
- cpudata->gmsr_misc_enable |=
- (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL);
+ if (flags & NVMM_X64_STATE_SEGS) {
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_CS);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_DS);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_ES);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_FS);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GS);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_SS);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GDT);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_IDT);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_LDT);
+ vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_TR);
+ }
- /* Must always be set. */
- vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE);
- vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE);
- cpudata->gxcr0 = XCR0_X87;
+ CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
+ if (flags & NVMM_X64_STATE_GPRS) {
+ memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs));
- /* Init XSAVE header. */
- cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
- cpudata->gfpu.xsh_xcomp_bv = 0;
+ vmx_vmwrite(VMCS_GUEST_RIP, state->gprs[NVMM_X64_GPR_RIP]);
+ vmx_vmwrite(VMCS_GUEST_RSP, state->gprs[NVMM_X64_GPR_RSP]);
+ vmx_vmwrite(VMCS_GUEST_RFLAGS, state->gprs[NVMM_X64_GPR_RFLAGS]);
+ }
- /* Set guest TSC to zero, more or less. */
- cpudata->tsc_offset = -cpu_counter();
+ if (flags & NVMM_X64_STATE_CRS) {
+ /*
+ * CR0_NE and CR4_VMXE are mandatory.
+ */
+ vmx_vmwrite(VMCS_GUEST_CR0,
+ state->crs[NVMM_X64_CR_CR0] | CR0_NE);
+ cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2];
+ vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE?
+ vmx_vmwrite(VMCS_GUEST_CR4,
+ state->crs[NVMM_X64_CR_CR4] | CR4_VMXE);
+ cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8];
- /* These MSRs are static. */
- cpudata->star = rdmsr(MSR_STAR);
- cpudata->cstar = rdmsr(MSR_CSTAR);
- cpudata->sfmask = rdmsr(MSR_SFMASK);
+ if (vmx_xcr0_mask != 0) {
+ /* Clear illegal XCR0 bits, set mandatory X87 bit. */
+ cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0];
+ cpudata->gxcr0 &= vmx_xcr0_mask;
+ cpudata->gxcr0 |= XCR0_X87;
+ }
+ }
- vmx_vmcs_leave(vcpu);
-}
+ CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
+ if (flags & NVMM_X64_STATE_DRS) {
+ memcpy(cpudata->drs, state->drs, sizeof(state->drs));
-static int
-vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
-{
- struct vmx_cpudata *cpudata;
- int error;
+ cpudata->drs[NVMM_X64_DR_DR6] &= 0xFFFFFFFF;
+ vmx_vmwrite(VMCS_GUEST_DR7, cpudata->drs[NVMM_X64_DR_DR7]);
+ }
- /* Allocate the VMX cpudata. */
- cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map,
- roundup(sizeof(*cpudata), PAGE_SIZE), 0,
- UVM_KMF_WIRED|UVM_KMF_ZERO);
- vcpu->cpudata = cpudata;
+ if (flags & NVMM_X64_STATE_MSRS) {
+ cpudata->gmsr[VMX_MSRLIST_STAR].val =
+ state->msrs[NVMM_X64_MSR_STAR];
+ cpudata->gmsr[VMX_MSRLIST_LSTAR].val =
+ state->msrs[NVMM_X64_MSR_LSTAR];
+ cpudata->gmsr[VMX_MSRLIST_CSTAR].val =
+ state->msrs[NVMM_X64_MSR_CSTAR];
+ cpudata->gmsr[VMX_MSRLIST_SFMASK].val =
+ state->msrs[NVMM_X64_MSR_SFMASK];
+ cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val =
+ state->msrs[NVMM_X64_MSR_KERNELGSBASE];
- /* VMCS */
- error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs,
- VMCS_NPAGES);
- if (error)
- goto error;
+ vmx_vmwrite(VMCS_GUEST_IA32_EFER,
+ state->msrs[NVMM_X64_MSR_EFER]);
+ vmx_vmwrite(VMCS_GUEST_IA32_PAT,
+ state->msrs[NVMM_X64_MSR_PAT]);
+ vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_CS,
+ state->msrs[NVMM_X64_MSR_SYSENTER_CS]);
+ vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_ESP,
+ state->msrs[NVMM_X64_MSR_SYSENTER_ESP]);
+ vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_EIP,
+ state->msrs[NVMM_X64_MSR_SYSENTER_EIP]);
- /* MSR Bitmap */
- error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
- MSRBM_NPAGES);
- if (error)
- goto error;
+ /* ENTRY_CTLS_LONG_MODE must match EFER_LMA. */
+ vmx_vmread(VMCS_ENTRY_CTLS, &ctls1);
+ if (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) {
+ ctls1 |= ENTRY_CTLS_LONG_MODE;
+ } else {
+ ctls1 &= ~ENTRY_CTLS_LONG_MODE;
+ }
+ vmx_vmwrite(VMCS_ENTRY_CTLS, ctls1);
+ }
- /* Guest MSR List */
- error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1);
- if (error)
- goto error;
+ if (flags & NVMM_X64_STATE_MISC) {
+ vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
+ intstate &= ~(INT_STATE_STI|INT_STATE_MOVSS);
+ if (state->misc[NVMM_X64_MISC_INT_SHADOW]) {
+ intstate |= INT_STATE_MOVSS;
+ }
+ vmx_vmwrite(VMCS_GUEST_INTERRUPTIBILITY, intstate);
- kcpuset_create(&cpudata->htlb_want_flush, true);
+ if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) {
+ vmx_event_waitexit_enable(vcpu, false);
+ } else {
+ vmx_event_waitexit_disable(vcpu, false);
+ }
- /* Init the VCPU info. */
- vmx_vcpu_init(mach, vcpu);
+ if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) {
+ vmx_event_waitexit_enable(vcpu, true);
+ } else {
+ vmx_event_waitexit_disable(vcpu, true);
+ }
+ }
- return 0;
+ CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
+ if (flags & NVMM_X64_STATE_FPU) {
+ memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu,
+ sizeof(state->fpu));
-error:
- if (cpudata->vmcs_pa) {
- vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs,
- VMCS_NPAGES);
- }
- if (cpudata->msrbm_pa) {
- vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
- MSRBM_NPAGES);
- }
- if (cpudata->gmsr_pa) {
- vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
+ fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave;
+ fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask;
+ fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask;
+
+ if (vmx_xcr0_mask != 0) {
+ /* Reset XSTATE_BV, to force a reload. */
+ cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
+ }
}
- kmem_free(cpudata, sizeof(*cpudata));
- return error;
+ vmx_vmcs_leave(vcpu);
}
static void
-vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
{
+ struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
struct vmx_cpudata *cpudata = vcpu->cpudata;
+ uint64_t intstate;
vmx_vmcs_enter(vcpu);
- vmx_asid_free(vcpu);
- vmx_vmcs_leave(vcpu);
- kcpuset_destroy(cpudata->htlb_want_flush);
+ if (flags & NVMM_X64_STATE_SEGS) {
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_CS);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_DS);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_ES);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_FS);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GS);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_SS);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GDT);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_IDT);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_LDT);
+ vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_TR);
+ }
- vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES);
- vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
- vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
- uvm_km_free(kernel_map, (vaddr_t)cpudata,
- roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
-}
+ CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
+ if (flags & NVMM_X64_STATE_GPRS) {
+ memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs));
-#define VMX_SEG_ATTRIB_TYPE __BITS(4,0)
-#define VMX_SEG_ATTRIB_DPL __BITS(6,5)
-#define VMX_SEG_ATTRIB_P __BIT(7)
-#define VMX_SEG_ATTRIB_AVL __BIT(12)
-#define VMX_SEG_ATTRIB_LONG __BIT(13)
-#define VMX_SEG_ATTRIB_DEF32 __BIT(14)
-#define VMX_SEG_ATTRIB_GRAN __BIT(15)
-#define VMX_SEG_ATTRIB_UNUSABLE __BIT(16)
+ vmx_vmread(VMCS_GUEST_RIP, &state->gprs[NVMM_X64_GPR_RIP]);
+ vmx_vmread(VMCS_GUEST_RSP, &state->gprs[NVMM_X64_GPR_RSP]);
+ vmx_vmread(VMCS_GUEST_RFLAGS, &state->gprs[NVMM_X64_GPR_RFLAGS]);
+ }
-static void
-vmx_vcpu_setstate_seg(struct nvmm_x64_state_seg *segs, int idx)
-{
- uint64_t attrib;
+ if (flags & NVMM_X64_STATE_CRS) {
+ vmx_vmread(VMCS_GUEST_CR0, &state->crs[NVMM_X64_CR_CR0]);
+ state->crs[NVMM_X64_CR_CR2] = cpudata->gcr2;
+ vmx_vmread(VMCS_GUEST_CR3, &state->crs[NVMM_X64_CR_CR3]);
+ vmx_vmread(VMCS_GUEST_CR4, &state->crs[NVMM_X64_CR_CR4]);
+ state->crs[NVMM_X64_CR_CR8] = cpudata->gcr8;
+ state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0;
- attrib =
- __SHIFTIN(segs[idx].attrib.type, VMX_SEG_ATTRIB_TYPE) |
- __SHIFTIN(segs[idx].attrib.dpl, VMX_SEG_ATTRIB_DPL) |
- __SHIFTIN(segs[idx].attrib.p, VMX_SEG_ATTRIB_P) |
- __SHIFTIN(segs[idx].attrib.avl, VMX_SEG_ATTRIB_AVL) |
- __SHIFTIN(segs[idx].attrib.lng, VMX_SEG_ATTRIB_LONG) |
- __SHIFTIN(segs[idx].attrib.def32, VMX_SEG_ATTRIB_DEF32) |
- __SHIFTIN(segs[idx].attrib.gran, VMX_SEG_ATTRIB_GRAN) |
- (!segs[idx].attrib.p ? VMX_SEG_ATTRIB_UNUSABLE : 0);
+ /* Hide VMXE. */
+ state->crs[NVMM_X64_CR_CR4] &= ~CR4_VMXE;
+ }
- if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) {
- vmx_vmwrite(vmx_guest_segs[idx].selector, segs[idx].selector);
- vmx_vmwrite(vmx_guest_segs[idx].attrib, attrib);
+ CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
+ if (flags & NVMM_X64_STATE_DRS) {
+ memcpy(state->drs, cpudata->drs, sizeof(state->drs));
+
+ vmx_vmread(VMCS_GUEST_DR7, &state->drs[NVMM_X64_DR_DR7]);
}
- vmx_vmwrite(vmx_guest_segs[idx].limit, segs[idx].limit);
- vmx_vmwrite(vmx_guest_segs[idx].base, segs[idx].base);
-}
-static void
-vmx_vcpu_getstate_seg(struct nvmm_x64_state_seg *segs, int idx)
-{
- uint64_t attrib = 0;
+ if (flags & NVMM_X64_STATE_MSRS) {
+ state->msrs[NVMM_X64_MSR_STAR] =
+ cpudata->gmsr[VMX_MSRLIST_STAR].val;
+ state->msrs[NVMM_X64_MSR_LSTAR] =
+ cpudata->gmsr[VMX_MSRLIST_LSTAR].val;
+ state->msrs[NVMM_X64_MSR_CSTAR] =
+ cpudata->gmsr[VMX_MSRLIST_CSTAR].val;
+ state->msrs[NVMM_X64_MSR_SFMASK] =
+ cpudata->gmsr[VMX_MSRLIST_SFMASK].val;
+ state->msrs[NVMM_X64_MSR_KERNELGSBASE] =
+ cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val;
- if (idx != NVMM_X64_SEG_GDT && idx != NVMM_X64_SEG_IDT) {
- vmx_vmread(vmx_guest_segs[idx].selector, &segs[idx].selector);
- vmx_vmread(vmx_guest_segs[idx].attrib, &attrib);
+ vmx_vmread(VMCS_GUEST_IA32_EFER,
+ &state->msrs[NVMM_X64_MSR_EFER]);
+ vmx_vmread(VMCS_GUEST_IA32_PAT,
+ &state->msrs[NVMM_X64_MSR_PAT]);
+ vmx_vmread(VMCS_GUEST_IA32_SYSENTER_CS,
+ &state->msrs[NVMM_X64_MSR_SYSENTER_CS]);
+ vmx_vmread(VMCS_GUEST_IA32_SYSENTER_ESP,
+ &state->msrs[NVMM_X64_MSR_SYSENTER_ESP]);
+ vmx_vmread(VMCS_GUEST_IA32_SYSENTER_EIP,
+ &state->msrs[NVMM_X64_MSR_SYSENTER_EIP]);
}
- vmx_vmread(vmx_guest_segs[idx].limit, &segs[idx].limit);
- vmx_vmread(vmx_guest_segs[idx].base, &segs[idx].base);
- segs[idx].attrib.type = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_TYPE);
- segs[idx].attrib.dpl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DPL);
- segs[idx].attrib.p = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_P);
- segs[idx].attrib.avl = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_AVL);
- segs[idx].attrib.lng = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_LONG);
- segs[idx].attrib.def32 = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_DEF32);
- segs[idx].attrib.gran = __SHIFTOUT(attrib, VMX_SEG_ATTRIB_GRAN);
- if (attrib & VMX_SEG_ATTRIB_UNUSABLE) {
- segs[idx].attrib.p = 0;
+ if (flags & NVMM_X64_STATE_MISC) {
+ vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
+ state->misc[NVMM_X64_MISC_INT_SHADOW] =
+ (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0;
+
+ state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] =
+ cpudata->int_window_exit;
+ state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] =
+ cpudata->nmi_window_exit;
}
+
+ CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
+ if (flags & NVMM_X64_STATE_FPU) {
+ memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave,
+ sizeof(state->fpu));
+ }
+
+ vmx_vmcs_leave(vcpu);
}
-static inline bool
-vmx_state_tlb_flush(struct nvmm_x64_state *state, uint64_t flags)
+/* -------------------------------------------------------------------------- */
+
+static void
+vmx_asid_alloc(struct nvmm_cpu *vcpu)
{
- uint64_t cr0, cr3, cr4, efer;
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ size_t i, oct, bit;
- if (flags & NVMM_X64_STATE_CRS) {
- vmx_vmread(VMCS_GUEST_CR0, &cr0);
- if ((cr0 ^ state->crs[NVMM_X64_CR_CR0]) & CR0_TLB_FLUSH) {
- return true;
- }
- vmx_vmread(VMCS_GUEST_CR3, &cr3);
- if (cr3 != state->crs[NVMM_X64_CR_CR3]) {
- return true;
- }
- vmx_vmread(VMCS_GUEST_CR4, &cr4);
- if ((cr4 ^ state->crs[NVMM_X64_CR_CR4]) & CR4_TLB_FLUSH) {
- return true;
- }
- }
+ mutex_enter(&vmx_asidlock);
- if (flags & NVMM_X64_STATE_MSRS) {
- vmx_vmread(VMCS_GUEST_IA32_EFER, &efer);
- if ((efer ^
- state->msrs[NVMM_X64_MSR_EFER]) & EFER_TLB_FLUSH) {
- return true;
+ for (i = 0; i < vmx_maxasid; i++) {
+ oct = i / 8;
+ bit = i % 8;
+
+ if (vmx_asidmap[oct] & __BIT(bit)) {
+ continue;
}
+
+ cpudata->asid = i;
+
+ vmx_asidmap[oct] |= __BIT(bit);
+ vmx_vmwrite(VMCS_VPID, i);
+ mutex_exit(&vmx_asidlock);
+ return;
}
- return false;
+ mutex_exit(&vmx_asidlock);
+
+ panic("%s: impossible", __func__);
}
static void
-vmx_vcpu_setstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
+vmx_asid_free(struct nvmm_cpu *vcpu)
{
- struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- struct fxsave *fpustate;
- uint64_t ctls1, intstate;
+ size_t oct, bit;
+ uint64_t asid;
- vmx_vmcs_enter(vcpu);
+ vmx_vmread(VMCS_VPID, &asid);
- if (vmx_state_tlb_flush(state, flags)) {
- cpudata->gtlb_want_flush = true;
- }
+ oct = asid / 8;
+ bit = asid % 8;
- if (flags & NVMM_X64_STATE_SEGS) {
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_CS);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_DS);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_ES);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_FS);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GS);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_SS);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_GDT);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_IDT);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_LDT);
- vmx_vcpu_setstate_seg(state->segs, NVMM_X64_SEG_TR);
- }
+ mutex_enter(&vmx_asidlock);
+ vmx_asidmap[oct] &= ~__BIT(bit);
+ mutex_exit(&vmx_asidlock);
+}
- CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
- if (flags & NVMM_X64_STATE_GPRS) {
- memcpy(cpudata->gprs, state->gprs, sizeof(state->gprs));
+static void
+vmx_vcpu_init(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ struct vmcs *vmcs = cpudata->vmcs;
+ struct msr_entry *gmsr = cpudata->gmsr;
+ extern uint8_t vmx_resume_rip;
+ uint64_t rev, eptp;
- vmx_vmwrite(VMCS_GUEST_RIP, state->gprs[NVMM_X64_GPR_RIP]);
- vmx_vmwrite(VMCS_GUEST_RSP, state->gprs[NVMM_X64_GPR_RSP]);
- vmx_vmwrite(VMCS_GUEST_RFLAGS, state->gprs[NVMM_X64_GPR_RFLAGS]);
- }
+ rev = vmx_get_revision();
- if (flags & NVMM_X64_STATE_CRS) {
- /* These bits are mandatory. */
- state->crs[NVMM_X64_CR_CR4] |= CR4_VMXE;
- state->crs[NVMM_X64_CR_CR0] |= CR0_NE;
+ memset(vmcs, 0, VMCS_SIZE);
+ vmcs->ident = __SHIFTIN(rev, VMCS_IDENT_REVISION);
+ vmcs->abort = 0;
- vmx_vmwrite(VMCS_GUEST_CR0, state->crs[NVMM_X64_CR_CR0]);
- cpudata->gcr2 = state->crs[NVMM_X64_CR_CR2];
- vmx_vmwrite(VMCS_GUEST_CR3, state->crs[NVMM_X64_CR_CR3]); // XXX PDPTE?
- vmx_vmwrite(VMCS_GUEST_CR4, state->crs[NVMM_X64_CR_CR4]);
- cpudata->gcr8 = state->crs[NVMM_X64_CR_CR8];
+ vmx_vmcs_enter(vcpu);
- if (vmx_xcr0_mask != 0) {
- /* Clear illegal XCR0 bits, set mandatory X87 bit. */
- cpudata->gxcr0 = state->crs[NVMM_X64_CR_XCR0];
- cpudata->gxcr0 &= vmx_xcr0_mask;
- cpudata->gxcr0 |= XCR0_X87;
- }
- }
+ /* No link pointer. */
+ vmx_vmwrite(VMCS_LINK_POINTER, 0xFFFFFFFFFFFFFFFF);
- CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
- if (flags & NVMM_X64_STATE_DRS) {
- memcpy(cpudata->drs, state->drs, sizeof(state->drs));
+ /* Install the CTLSs. */
+ vmx_vmwrite(VMCS_PINBASED_CTLS, vmx_pinbased_ctls);
+ vmx_vmwrite(VMCS_PROCBASED_CTLS, vmx_procbased_ctls);
+ vmx_vmwrite(VMCS_PROCBASED_CTLS2, vmx_procbased_ctls2);
+ vmx_vmwrite(VMCS_ENTRY_CTLS, vmx_entry_ctls);
+ vmx_vmwrite(VMCS_EXIT_CTLS, vmx_exit_ctls);
- cpudata->drs[NVMM_X64_DR_DR6] &= 0xFFFFFFFF;
- vmx_vmwrite(VMCS_GUEST_DR7, cpudata->drs[NVMM_X64_DR_DR7]);
- }
+ /* Allow direct access to certain MSRs. */
+ memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_CS, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_ESP, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_SYSENTER_EIP, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_FSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_GSBASE, true, true);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_TSC, true, false);
+ vmx_vcpu_msr_allow(cpudata->msrbm, MSR_IA32_ARCH_CAPABILITIES,
+ true, false);
+ vmx_vmwrite(VMCS_MSR_BITMAP, (uint64_t)cpudata->msrbm_pa);
+
+ /*
+ * List of Guest MSRs loaded on VMENTRY, saved on VMEXIT. This
+ * includes the L1D_FLUSH MSR, to mitigate L1TF.
+ */
+ gmsr[VMX_MSRLIST_STAR].msr = MSR_STAR;
+ gmsr[VMX_MSRLIST_STAR].val = 0;
+ gmsr[VMX_MSRLIST_LSTAR].msr = MSR_LSTAR;
+ gmsr[VMX_MSRLIST_LSTAR].val = 0;
+ gmsr[VMX_MSRLIST_CSTAR].msr = MSR_CSTAR;
+ gmsr[VMX_MSRLIST_CSTAR].val = 0;
+ gmsr[VMX_MSRLIST_SFMASK].msr = MSR_SFMASK;
+ gmsr[VMX_MSRLIST_SFMASK].val = 0;
+ gmsr[VMX_MSRLIST_KERNELGSBASE].msr = MSR_KERNELGSBASE;
+ gmsr[VMX_MSRLIST_KERNELGSBASE].val = 0;
+ gmsr[VMX_MSRLIST_L1DFLUSH].msr = MSR_IA32_FLUSH_CMD;
+ gmsr[VMX_MSRLIST_L1DFLUSH].val = IA32_FLUSH_CMD_L1D_FLUSH;
+ vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_ADDRESS, cpudata->gmsr_pa);
+ vmx_vmwrite(VMCS_EXIT_MSR_STORE_ADDRESS, cpudata->gmsr_pa);
+ vmx_vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT, vmx_msrlist_entry_nmsr);
+ vmx_vmwrite(VMCS_EXIT_MSR_STORE_COUNT, VMX_MSRLIST_EXIT_NMSR);
- if (flags & NVMM_X64_STATE_MSRS) {
- cpudata->gmsr[VMX_MSRLIST_STAR].val =
- state->msrs[NVMM_X64_MSR_STAR];
- cpudata->gmsr[VMX_MSRLIST_LSTAR].val =
- state->msrs[NVMM_X64_MSR_LSTAR];
- cpudata->gmsr[VMX_MSRLIST_CSTAR].val =
- state->msrs[NVMM_X64_MSR_CSTAR];
- cpudata->gmsr[VMX_MSRLIST_SFMASK].val =
- state->msrs[NVMM_X64_MSR_SFMASK];
- cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val =
- state->msrs[NVMM_X64_MSR_KERNELGSBASE];
+ /* Force CR0_NW and CR0_CD to zero, CR0_ET to one. */
+ vmx_vmwrite(VMCS_CR0_MASK, CR0_NW|CR0_CD|CR0_ET);
+ vmx_vmwrite(VMCS_CR0_SHADOW, CR0_ET);
- vmx_vmwrite(VMCS_GUEST_IA32_EFER,
- state->msrs[NVMM_X64_MSR_EFER]);
- vmx_vmwrite(VMCS_GUEST_IA32_PAT,
- state->msrs[NVMM_X64_MSR_PAT]);
- vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_CS,
- state->msrs[NVMM_X64_MSR_SYSENTER_CS]);
- vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_ESP,
- state->msrs[NVMM_X64_MSR_SYSENTER_ESP]);
- vmx_vmwrite(VMCS_GUEST_IA32_SYSENTER_EIP,
- state->msrs[NVMM_X64_MSR_SYSENTER_EIP]);
+ /* Force CR4_VMXE to zero. */
+ vmx_vmwrite(VMCS_CR4_MASK, CR4_VMXE);
- /* ENTRY_CTLS_LONG_MODE must match EFER_LMA. */
- vmx_vmread(VMCS_ENTRY_CTLS, &ctls1);
- if (state->msrs[NVMM_X64_MSR_EFER] & EFER_LMA) {
- ctls1 |= ENTRY_CTLS_LONG_MODE;
- } else {
- ctls1 &= ~ENTRY_CTLS_LONG_MODE;
- }
- vmx_vmwrite(VMCS_ENTRY_CTLS, ctls1);
- }
+ /* Set the Host state for resuming. */
+ vmx_vmwrite(VMCS_HOST_RIP, (uint64_t)&vmx_resume_rip);
+ vmx_vmwrite(VMCS_HOST_CS_SELECTOR, GSEL(GCODE_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_SS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_DS_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_ES_SELECTOR, GSEL(GDATA_SEL, SEL_KPL));
+ vmx_vmwrite(VMCS_HOST_FS_SELECTOR, 0);
+ vmx_vmwrite(VMCS_HOST_GS_SELECTOR, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_CS, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_ESP, 0);
+ vmx_vmwrite(VMCS_HOST_IA32_SYSENTER_EIP, 0);
+ vmx_vmwrite(VMCS_HOST_IDTR_BASE, (uint64_t)idt);
+ vmx_vmwrite(VMCS_HOST_IA32_PAT, rdmsr(MSR_CR_PAT));
+ vmx_vmwrite(VMCS_HOST_IA32_EFER, rdmsr(MSR_EFER));
+ vmx_vmwrite(VMCS_HOST_CR0, rcr0());
- if (flags & NVMM_X64_STATE_MISC) {
- vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
- intstate &= ~(INT_STATE_STI|INT_STATE_MOVSS);
- if (state->misc[NVMM_X64_MISC_INT_SHADOW]) {
- intstate |= INT_STATE_MOVSS;
- }
- vmx_vmwrite(VMCS_GUEST_INTERRUPTIBILITY, intstate);
+ /* Generate ASID. */
+ vmx_asid_alloc(vcpu);
- if (state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT]) {
- vmx_event_waitexit_enable(vcpu, false);
- } else {
- vmx_event_waitexit_disable(vcpu, false);
- }
+ /* Enable Extended Paging, 4-Level. */
+ eptp =
+ __SHIFTIN(vmx_eptp_type, EPTP_TYPE) |
+ __SHIFTIN(4-1, EPTP_WALKLEN) |
+ EPTP_FLAGS_AD |
+ mach->vm->vm_map.pmap->pm_pdirpa[0];
+ vmx_vmwrite(VMCS_EPTP, eptp);
- if (state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT]) {
- vmx_event_waitexit_enable(vcpu, true);
- } else {
- vmx_event_waitexit_disable(vcpu, true);
- }
- }
+ /* Init IA32_MISC_ENABLE. */
+ cpudata->gmsr_misc_enable = rdmsr(MSR_MISC_ENABLE);
+ cpudata->gmsr_misc_enable &=
+ ~(IA32_MISC_PERFMON_EN|IA32_MISC_EISST_EN|IA32_MISC_MWAIT_EN);
+ cpudata->gmsr_misc_enable |=
+ (IA32_MISC_BTS_UNAVAIL|IA32_MISC_PEBS_UNAVAIL);
- CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
- if (flags & NVMM_X64_STATE_FPU) {
- memcpy(cpudata->gfpu.xsh_fxsave, &state->fpu,
- sizeof(state->fpu));
+ /* Must always be set. */
+ vmx_vmwrite(VMCS_GUEST_CR4, CR4_VMXE);
+ vmx_vmwrite(VMCS_GUEST_CR0, CR0_NE);
+ cpudata->gxcr0 = XCR0_X87;
- fpustate = (struct fxsave *)cpudata->gfpu.xsh_fxsave;
- fpustate->fx_mxcsr_mask &= x86_fpu_mxcsr_mask;
- fpustate->fx_mxcsr &= fpustate->fx_mxcsr_mask;
+ /* Init XSAVE header. */
+ cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
+ cpudata->gfpu.xsh_xcomp_bv = 0;
- if (vmx_xcr0_mask != 0) {
- /* Reset XSTATE_BV, to force a reload. */
- cpudata->gfpu.xsh_xstate_bv = vmx_xcr0_mask;
- }
- }
+ /* Set guest TSC to zero, more or less. */
+ cpudata->tsc_offset = -cpu_counter();
+
+ /* These MSRs are static. */
+ cpudata->star = rdmsr(MSR_STAR);
+ cpudata->cstar = rdmsr(MSR_CSTAR);
+ cpudata->sfmask = rdmsr(MSR_SFMASK);
vmx_vmcs_leave(vcpu);
}
-static void
-vmx_vcpu_getstate(struct nvmm_cpu *vcpu, void *data, uint64_t flags)
+static int
+vmx_vcpu_create(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
{
- struct nvmm_x64_state *state = (struct nvmm_x64_state *)data;
- struct vmx_cpudata *cpudata = vcpu->cpudata;
- uint64_t intstate;
+ struct vmx_cpudata *cpudata;
+ int error;
- vmx_vmcs_enter(vcpu);
+ /* Allocate the VMX cpudata. */
+ cpudata = (struct vmx_cpudata *)uvm_km_alloc(kernel_map,
+ roundup(sizeof(*cpudata), PAGE_SIZE), 0,
+ UVM_KMF_WIRED|UVM_KMF_ZERO);
+ vcpu->cpudata = cpudata;
- if (flags & NVMM_X64_STATE_SEGS) {
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_CS);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_DS);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_ES);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_FS);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GS);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_SS);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_GDT);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_IDT);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_LDT);
- vmx_vcpu_getstate_seg(state->segs, NVMM_X64_SEG_TR);
- }
+ /* VMCS */
+ error = vmx_memalloc(&cpudata->vmcs_pa, (vaddr_t *)&cpudata->vmcs,
+ VMCS_NPAGES);
+ if (error)
+ goto error;
- CTASSERT(sizeof(cpudata->gprs) == sizeof(state->gprs));
- if (flags & NVMM_X64_STATE_GPRS) {
- memcpy(state->gprs, cpudata->gprs, sizeof(state->gprs));
+ /* MSR Bitmap */
+ error = vmx_memalloc(&cpudata->msrbm_pa, (vaddr_t *)&cpudata->msrbm,
+ MSRBM_NPAGES);
+ if (error)
+ goto error;
- vmx_vmread(VMCS_GUEST_RIP, &state->gprs[NVMM_X64_GPR_RIP]);
- vmx_vmread(VMCS_GUEST_RSP, &state->gprs[NVMM_X64_GPR_RSP]);
- vmx_vmread(VMCS_GUEST_RFLAGS, &state->gprs[NVMM_X64_GPR_RFLAGS]);
- }
+ /* Guest MSR List */
+ error = vmx_memalloc(&cpudata->gmsr_pa, (vaddr_t *)&cpudata->gmsr, 1);
+ if (error)
+ goto error;
- if (flags & NVMM_X64_STATE_CRS) {
- vmx_vmread(VMCS_GUEST_CR0, &state->crs[NVMM_X64_CR_CR0]);
- state->crs[NVMM_X64_CR_CR2] = cpudata->gcr2;
- vmx_vmread(VMCS_GUEST_CR3, &state->crs[NVMM_X64_CR_CR3]);
- vmx_vmread(VMCS_GUEST_CR4, &state->crs[NVMM_X64_CR_CR4]);
- state->crs[NVMM_X64_CR_CR8] = cpudata->gcr8;
- state->crs[NVMM_X64_CR_XCR0] = cpudata->gxcr0;
+ kcpuset_create(&cpudata->htlb_want_flush, true);
- /* Hide VMXE. */
- state->crs[NVMM_X64_CR_CR4] &= ~CR4_VMXE;
- }
+ /* Init the VCPU info. */
+ vmx_vcpu_init(mach, vcpu);
- CTASSERT(sizeof(cpudata->drs) == sizeof(state->drs));
- if (flags & NVMM_X64_STATE_DRS) {
- memcpy(state->drs, cpudata->drs, sizeof(state->drs));
+ return 0;
- vmx_vmread(VMCS_GUEST_DR7, &state->drs[NVMM_X64_DR_DR7]);
+error:
+ if (cpudata->vmcs_pa) {
+ vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs,
+ VMCS_NPAGES);
}
-
- if (flags & NVMM_X64_STATE_MSRS) {
- state->msrs[NVMM_X64_MSR_STAR] =
- cpudata->gmsr[VMX_MSRLIST_STAR].val;
- state->msrs[NVMM_X64_MSR_LSTAR] =
- cpudata->gmsr[VMX_MSRLIST_LSTAR].val;
- state->msrs[NVMM_X64_MSR_CSTAR] =
- cpudata->gmsr[VMX_MSRLIST_CSTAR].val;
- state->msrs[NVMM_X64_MSR_SFMASK] =
- cpudata->gmsr[VMX_MSRLIST_SFMASK].val;
- state->msrs[NVMM_X64_MSR_KERNELGSBASE] =
- cpudata->gmsr[VMX_MSRLIST_KERNELGSBASE].val;
-
- vmx_vmread(VMCS_GUEST_IA32_EFER,
- &state->msrs[NVMM_X64_MSR_EFER]);
- vmx_vmread(VMCS_GUEST_IA32_PAT,
- &state->msrs[NVMM_X64_MSR_PAT]);
- vmx_vmread(VMCS_GUEST_IA32_SYSENTER_CS,
- &state->msrs[NVMM_X64_MSR_SYSENTER_CS]);
- vmx_vmread(VMCS_GUEST_IA32_SYSENTER_ESP,
- &state->msrs[NVMM_X64_MSR_SYSENTER_ESP]);
- vmx_vmread(VMCS_GUEST_IA32_SYSENTER_EIP,
- &state->msrs[NVMM_X64_MSR_SYSENTER_EIP]);
+ if (cpudata->msrbm_pa) {
+ vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm,
+ MSRBM_NPAGES);
}
-
- if (flags & NVMM_X64_STATE_MISC) {
- vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY, &intstate);
- state->misc[NVMM_X64_MISC_INT_SHADOW] =
- (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0;
-
- state->misc[NVMM_X64_MISC_INT_WINDOW_EXIT] =
- cpudata->int_window_exit;
- state->misc[NVMM_X64_MISC_NMI_WINDOW_EXIT] =
- cpudata->nmi_window_exit;
+ if (cpudata->gmsr_pa) {
+ vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
}
- CTASSERT(sizeof(cpudata->gfpu.xsh_fxsave) == sizeof(state->fpu));
- if (flags & NVMM_X64_STATE_FPU) {
- memcpy(&state->fpu, cpudata->gfpu.xsh_fxsave,
- sizeof(state->fpu));
- }
+ kmem_free(cpudata, sizeof(*cpudata));
+ return error;
+}
+
+static void
+vmx_vcpu_destroy(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+{
+ struct vmx_cpudata *cpudata = vcpu->cpudata;
+ vmx_vmcs_enter(vcpu);
+ vmx_asid_free(vcpu);
vmx_vmcs_leave(vcpu);
+
+ kcpuset_destroy(cpudata->htlb_want_flush);
+
+ vmx_memfree(cpudata->vmcs_pa, (vaddr_t)cpudata->vmcs, VMCS_NPAGES);
+ vmx_memfree(cpudata->msrbm_pa, (vaddr_t)cpudata->msrbm, MSRBM_NPAGES);
+ vmx_memfree(cpudata->gmsr_pa, (vaddr_t)cpudata->gmsr, 1);
+ uvm_km_free(kernel_map, (vaddr_t)cpudata,
+ roundup(sizeof(*cpudata), PAGE_SIZE), UVM_KMF_WIRED);
}
/* -------------------------------------------------------------------------- */
@@ -2748,6 +2736,21 @@ vmx_ident(void)
}
static void
+vmx_init_asid(uint32_t maxasid)
+{
+ size_t allocsz;
+
+ mutex_init(&vmx_asidlock, MUTEX_DEFAULT, IPL_NONE);
+
+ vmx_maxasid = maxasid;
+ allocsz = roundup(maxasid, 8) / 8;
+ vmx_asidmap = kmem_zalloc(allocsz, KM_SLEEP);
+
+ /* ASID 0 is reserved for the host. */
+ vmx_asidmap[0] |= __BIT(0);
+}
+
+static void
vmx_change_cpu(void *arg1, void *arg2)
{
struct cpu_info *ci = curcpu();
