Module Name: src
Committed By: maxv
Date: Wed May 1 09:20:21 UTC 2019
Modified Files:
src/lib/libnvmm: libnvmm.c
src/sys/dev/nvmm: nvmm.c nvmm.h nvmm_internal.h nvmm_ioctl.h
src/sys/dev/nvmm/x86: nvmm_x86.h nvmm_x86_svm.c nvmm_x86_vmx.c
Log Message:
Use the comm page to inject events, rather than ioctls, and commit them in
vcpu_run. This saves a few syscalls and copyins.
For example on Windows 10, moving the mouse from the left to right sides of
the screen generates ~500 events, which now don't result in syscalls.
The error handling is done in vcpu_run and it is less precise, but this
doesn't matter a lot, and will be solved with future NVMM error codes.
To generate a diff of this commit:
cvs rdiff -u -r1.11 -r1.12 src/lib/libnvmm/libnvmm.c
cvs rdiff -u -r1.19 -r1.20 src/sys/dev/nvmm/nvmm.c
cvs rdiff -u -r1.8 -r1.9 src/sys/dev/nvmm/nvmm.h
cvs rdiff -u -r1.10 -r1.11 src/sys/dev/nvmm/nvmm_internal.h
cvs rdiff -u -r1.6 -r1.7 src/sys/dev/nvmm/nvmm_ioctl.h
cvs rdiff -u -r1.13 -r1.14 src/sys/dev/nvmm/x86/nvmm_x86.h
cvs rdiff -u -r1.44 -r1.45 src/sys/dev/nvmm/x86/nvmm_x86_svm.c
cvs rdiff -u -r1.32 -r1.33 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/lib/libnvmm/libnvmm.c
diff -u src/lib/libnvmm/libnvmm.c:1.11 src/lib/libnvmm/libnvmm.c:1.12
--- src/lib/libnvmm/libnvmm.c:1.11 Mon Apr 29 17:27:57 2019
+++ src/lib/libnvmm/libnvmm.c Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: libnvmm.c,v 1.11 2019/04/29 17:27:57 maxv Exp $ */
+/* $NetBSD: libnvmm.c,v 1.12 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -362,16 +362,15 @@ int
nvmm_vcpu_inject(struct nvmm_machine *mach, nvmm_cpuid_t cpuid,
struct nvmm_event *event)
{
- struct nvmm_ioc_vcpu_inject args;
- int ret;
-
- args.machid = mach->machid;
- args.cpuid = cpuid;
- memcpy(&args.event, event, sizeof(args.event));
+ struct nvmm_comm_page *comm;
- ret = ioctl(nvmm_fd, NVMM_IOC_VCPU_INJECT, &args);
- if (ret == -1)
+ if (__predict_false(cpuid >= mach->npages)) {
return -1;
+ }
+ comm = mach->pages[cpuid];
+
+ memcpy(&comm->event, event, sizeof(comm->event));
+ comm->event_commit = true;
return 0;
}
Index: src/sys/dev/nvmm/nvmm.c
diff -u src/sys/dev/nvmm/nvmm.c:1.19 src/sys/dev/nvmm/nvmm.c:1.20
--- src/sys/dev/nvmm/nvmm.c:1.19 Sun Apr 28 14:22:13 2019
+++ src/sys/dev/nvmm/nvmm.c Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm.c,v 1.19 2019/04/28 14:22:13 maxv Exp $ */
+/* $NetBSD: nvmm.c,v 1.20 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.19 2019/04/28 14:22:13 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.20 2019/05/01 09:20:21 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -503,7 +503,7 @@ nvmm_vcpu_inject(struct nvmm_owner *owne
if (error)
goto out;
- error = (*nvmm_impl->vcpu_inject)(mach, vcpu, &args->event);
+ error = (*nvmm_impl->vcpu_inject)(vcpu);
nvmm_vcpu_put(vcpu);
out:
Index: src/sys/dev/nvmm/nvmm.h
diff -u src/sys/dev/nvmm/nvmm.h:1.8 src/sys/dev/nvmm/nvmm.h:1.9
--- src/sys/dev/nvmm/nvmm.h:1.8 Sun Apr 28 14:22:13 2019
+++ src/sys/dev/nvmm/nvmm.h Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm.h,v 1.8 2019/04/28 14:22:13 maxv Exp $ */
+/* $NetBSD: nvmm.h,v 1.9 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -94,6 +94,18 @@ struct nvmm_capability {
struct nvmm_cap_md arch;
};
+struct nvmm_comm_page {
+ /* State. */
+ uint64_t state_wanted;
+ uint64_t state_cached;
+ uint64_t state_commit;
+ struct nvmm_vcpu_state state;
+
+ /* Event. */
+ bool event_commit;
+ struct nvmm_event event;
+};
+
/*
* Bits 20:27 -> machid
* Bits 12:19 -> cpuid
Index: src/sys/dev/nvmm/nvmm_internal.h
diff -u src/sys/dev/nvmm/nvmm_internal.h:1.10 src/sys/dev/nvmm/nvmm_internal.h:1.11
--- src/sys/dev/nvmm/nvmm_internal.h:1.10 Sun Apr 28 14:22:13 2019
+++ src/sys/dev/nvmm/nvmm_internal.h Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_internal.h,v 1.10 2019/04/28 14:22:13 maxv Exp $ */
+/* $NetBSD: nvmm_internal.h,v 1.11 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -107,8 +107,7 @@ struct nvmm_impl {
void (*vcpu_destroy)(struct nvmm_machine *, struct nvmm_cpu *);
void (*vcpu_setstate)(struct nvmm_cpu *);
void (*vcpu_getstate)(struct nvmm_cpu *);
- int (*vcpu_inject)(struct nvmm_machine *, struct nvmm_cpu *,
- struct nvmm_event *);
+ int (*vcpu_inject)(struct nvmm_cpu *);
int (*vcpu_run)(struct nvmm_machine *, struct nvmm_cpu *,
struct nvmm_exit *);
};
Index: src/sys/dev/nvmm/nvmm_ioctl.h
diff -u src/sys/dev/nvmm/nvmm_ioctl.h:1.6 src/sys/dev/nvmm/nvmm_ioctl.h:1.7
--- src/sys/dev/nvmm/nvmm_ioctl.h:1.6 Sun Apr 28 14:22:13 2019
+++ src/sys/dev/nvmm/nvmm_ioctl.h Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_ioctl.h,v 1.6 2019/04/28 14:22:13 maxv Exp $ */
+/* $NetBSD: nvmm_ioctl.h,v 1.7 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -75,7 +75,6 @@ struct nvmm_ioc_vcpu_getstate {
struct nvmm_ioc_vcpu_inject {
nvmm_machid_t machid;
nvmm_cpuid_t cpuid;
- struct nvmm_event event;
};
struct nvmm_ioc_vcpu_run {
Index: src/sys/dev/nvmm/x86/nvmm_x86.h
diff -u src/sys/dev/nvmm/x86/nvmm_x86.h:1.13 src/sys/dev/nvmm/x86/nvmm_x86.h:1.14
--- src/sys/dev/nvmm/x86/nvmm_x86.h:1.13 Sun Apr 28 14:22:13 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86.h Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86.h,v 1.13 2019/04/28 14:22:13 maxv Exp $ */
+/* $NetBSD: nvmm_x86.h,v 1.14 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -229,6 +229,8 @@ struct nvmm_x64_state {
struct fxsave fpu;
};
+#define nvmm_vcpu_state nvmm_x64_state
+
#define NVMM_X86_CONF_CPUID 0
#define NVMM_X86_NCONF 1
@@ -248,13 +250,6 @@ struct nvmm_x86_conf_cpuid {
} del;
};
-struct nvmm_comm_page {
- uint64_t state_wanted;
- uint64_t state_cached;
- uint64_t state_commit;
- struct nvmm_x64_state state;
-};
-
#ifdef _KERNEL
struct nvmm_x86_cpuid_mask {
uint32_t eax;
Index: src/sys/dev/nvmm/x86/nvmm_x86_svm.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.44 src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.45
--- src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.44 Mon Apr 29 18:54:25 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_svm.c Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_svm.c,v 1.44 2019/04/29 18:54:25 maxv Exp $ */
+/* $NetBSD: nvmm_x86_svm.c,v 1.45 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.44 2019/04/29 18:54:25 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.45 2019/05/01 09:20:21 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -661,46 +661,51 @@ svm_event_has_error(uint64_t vector)
}
static int
-svm_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
- struct nvmm_event *event)
+svm_vcpu_inject(struct nvmm_cpu *vcpu)
{
+ struct nvmm_comm_page *comm = vcpu->comm;
struct svm_cpudata *cpudata = vcpu->cpudata;
struct vmcb *vmcb = cpudata->vmcb;
+ enum nvmm_event_type evtype;
+ uint64_t vector, error;
int type = 0, err = 0;
- if (event->vector >= 256) {
+ evtype = comm->event.type;
+ vector = comm->event.vector;
+ error = comm->event.u.error;
+ __insn_barrier();
+
+ if (__predict_false(vector >= 256)) {
return EINVAL;
}
- switch (event->type) {
+ switch (evtype) {
case NVMM_EVENT_INTERRUPT_HW:
type = SVM_EVENT_TYPE_HW_INT;
- if (event->vector == 2) {
+ if (vector == 2) {
type = SVM_EVENT_TYPE_NMI;
svm_event_waitexit_enable(vcpu, true);
}
err = 0;
break;
- case NVMM_EVENT_INTERRUPT_SW:
- return EINVAL;
case NVMM_EVENT_EXCEPTION:
type = SVM_EVENT_TYPE_EXC;
- if (event->vector == 2 || event->vector >= 32)
+ if (vector == 2 || vector >= 32)
return EINVAL;
- if (event->vector == 3 || event->vector == 0)
+ if (vector == 3 || vector == 0)
return EINVAL;
- err = svm_event_has_error(event->vector);
+ err = svm_event_has_error(vector);
break;
default:
return EINVAL;
}
vmcb->ctrl.eventinj =
- __SHIFTIN(event->vector, VMCB_CTRL_EVENTINJ_VECTOR) |
+ __SHIFTIN(vector, VMCB_CTRL_EVENTINJ_VECTOR) |
__SHIFTIN(type, VMCB_CTRL_EVENTINJ_TYPE) |
__SHIFTIN(err, VMCB_CTRL_EVENTINJ_EV) |
__SHIFTIN(1, VMCB_CTRL_EVENTINJ_V) |
- __SHIFTIN(event->u.error, VMCB_CTRL_EVENTINJ_ERRORCODE);
+ __SHIFTIN(error, VMCB_CTRL_EVENTINJ_ERRORCODE);
cpudata->evt_pending = true;
@@ -708,33 +713,43 @@ svm_vcpu_inject(struct nvmm_machine *mac
}
static void
-svm_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+svm_inject_ud(struct nvmm_cpu *vcpu)
{
- struct nvmm_event event;
+ struct nvmm_comm_page *comm = vcpu->comm;
int ret __diagused;
- event.type = NVMM_EVENT_EXCEPTION;
- event.vector = 6;
- event.u.error = 0;
+ comm->event.type = NVMM_EVENT_EXCEPTION;
+ comm->event.vector = 6;
+ comm->event.u.error = 0;
- ret = svm_vcpu_inject(mach, vcpu, &event);
+ ret = svm_vcpu_inject(vcpu);
KASSERT(ret == 0);
}
static void
-svm_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+svm_inject_gp(struct nvmm_cpu *vcpu)
{
- struct nvmm_event event;
+ struct nvmm_comm_page *comm = vcpu->comm;
int ret __diagused;
- event.type = NVMM_EVENT_EXCEPTION;
- event.vector = 13;
- event.u.error = 0;
+ comm->event.type = NVMM_EVENT_EXCEPTION;
+ comm->event.vector = 13;
+ comm->event.u.error = 0;
- ret = svm_vcpu_inject(mach, vcpu, &event);
+ ret = svm_vcpu_inject(vcpu);
KASSERT(ret == 0);
}
+static inline int
+svm_vcpu_event_commit(struct nvmm_cpu *vcpu)
+{
+ if (__predict_true(!vcpu->comm->event_commit)) {
+ return 0;
+ }
+ vcpu->comm->event_commit = false;
+ return svm_vcpu_inject(vcpu);
+}
+
static inline void
svm_inkernel_advance(struct vmcb *vmcb)
{
@@ -1018,7 +1033,7 @@ handled:
return true;
error:
- svm_inject_gp(mach, vcpu);
+ svm_inject_gp(vcpu);
return true;
}
@@ -1117,7 +1132,7 @@ svm_exit_xsetbv(struct nvmm_machine *mac
return;
error:
- svm_inject_gp(mach, vcpu);
+ svm_inject_gp(vcpu);
}
static void
@@ -1283,6 +1298,9 @@ svm_vcpu_run(struct nvmm_machine *mach,
uint64_t machgen;
int hcpu, s;
+ if (__predict_false(svm_vcpu_event_commit(vcpu) != 0)) {
+ return EINVAL;
+ }
svm_vcpu_state_commit(vcpu);
comm->state_cached = 0;
@@ -1368,7 +1386,7 @@ svm_vcpu_run(struct nvmm_machine *mach,
case VMCB_EXITCODE_CLGI:
case VMCB_EXITCODE_SKINIT:
case VMCB_EXITCODE_RDTSCP:
- svm_inject_ud(mach, vcpu);
+ svm_inject_ud(vcpu);
exit->reason = NVMM_EXIT_NONE;
break;
case VMCB_EXITCODE_MONITOR:
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.32 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.33
--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.32 Mon Apr 29 18:54:26 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c Wed May 1 09:20:21 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.32 2019/04/29 18:54:26 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.33 2019/05/01 09:20:21 maxv Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.32 2019/04/29 18:54:26 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.33 2019/05/01 09:20:21 maxv Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -992,49 +992,53 @@ vmx_event_has_error(uint64_t vector)
}
static int
-vmx_vcpu_inject(struct nvmm_machine *mach, struct nvmm_cpu *vcpu,
- struct nvmm_event *event)
+vmx_vcpu_inject(struct nvmm_cpu *vcpu)
{
+ struct nvmm_comm_page *comm = vcpu->comm;
struct vmx_cpudata *cpudata = vcpu->cpudata;
int type = 0, err = 0, ret = EINVAL;
- uint64_t info;
+ enum nvmm_event_type evtype;
+ uint64_t info, vector, error;
+
+ evtype = comm->event.type;
+ vector = comm->event.vector;
+ error = comm->event.u.error;
+ __insn_barrier();
- if (event->vector >= 256) {
+ if (__predict_false(vector >= 256)) {
return EINVAL;
}
vmx_vmcs_enter(vcpu);
- switch (event->type) {
+ switch (evtype) {
case NVMM_EVENT_INTERRUPT_HW:
type = INTR_TYPE_EXT_INT;
- if (event->vector == 2) {
+ if (vector == 2) {
type = INTR_TYPE_NMI;
vmx_event_waitexit_enable(vcpu, true);
}
err = 0;
break;
- case NVMM_EVENT_INTERRUPT_SW:
- goto out;
case NVMM_EVENT_EXCEPTION:
- if (event->vector == 2 || event->vector >= 32)
+ if (vector == 2 || vector >= 32)
goto out;
- if (event->vector == 3 || event->vector == 0)
+ if (vector == 3 || vector == 0)
goto out;
type = INTR_TYPE_HW_EXC;
- err = vmx_event_has_error(event->vector);
+ err = vmx_event_has_error(vector);
break;
default:
goto out;
}
info =
- __SHIFTIN(event->vector, INTR_INFO_VECTOR) |
+ __SHIFTIN(vector, INTR_INFO_VECTOR) |
__SHIFTIN(type, INTR_INFO_TYPE) |
__SHIFTIN(err, INTR_INFO_ERROR) |
__SHIFTIN(1, INTR_INFO_VALID);
vmx_vmwrite(VMCS_ENTRY_INTR_INFO, info);
- vmx_vmwrite(VMCS_ENTRY_EXCEPTION_ERROR, event->u.error);
+ vmx_vmwrite(VMCS_ENTRY_EXCEPTION_ERROR, error);
cpudata->evt_pending = true;
ret = 0;
@@ -1045,33 +1049,43 @@ out:
}
static void
-vmx_inject_ud(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+vmx_inject_ud(struct nvmm_cpu *vcpu)
{
- struct nvmm_event event;
+ struct nvmm_comm_page *comm = vcpu->comm;
int ret __diagused;
- event.type = NVMM_EVENT_EXCEPTION;
- event.vector = 6;
- event.u.error = 0;
+ comm->event.type = NVMM_EVENT_EXCEPTION;
+ comm->event.vector = 6;
+ comm->event.u.error = 0;
- ret = vmx_vcpu_inject(mach, vcpu, &event);
+ ret = vmx_vcpu_inject(vcpu);
KASSERT(ret == 0);
}
static void
-vmx_inject_gp(struct nvmm_machine *mach, struct nvmm_cpu *vcpu)
+vmx_inject_gp(struct nvmm_cpu *vcpu)
{
- struct nvmm_event event;
+ struct nvmm_comm_page *comm = vcpu->comm;
int ret __diagused;
- event.type = NVMM_EVENT_EXCEPTION;
- event.vector = 13;
- event.u.error = 0;
+ comm->event.type = NVMM_EVENT_EXCEPTION;
+ comm->event.vector = 13;
+ comm->event.u.error = 0;
- ret = vmx_vcpu_inject(mach, vcpu, &event);
+ ret = vmx_vcpu_inject(vcpu);
KASSERT(ret == 0);
}
+static inline int
+vmx_vcpu_event_commit(struct nvmm_cpu *vcpu)
+{
+ if (__predict_true(!vcpu->comm->event_commit)) {
+ return 0;
+ }
+ vcpu->comm->event_commit = false;
+ return vmx_vcpu_inject(vcpu);
+}
+
static inline void
vmx_inkernel_advance(void)
{
@@ -1430,7 +1444,7 @@ vmx_exit_cr(struct nvmm_machine *mach, s
}
if (ret == -1) {
- vmx_inject_gp(mach, vcpu);
+ vmx_inject_gp(vcpu);
}
exit->reason = NVMM_EXIT_NONE;
@@ -1575,7 +1589,7 @@ handled:
return true;
error:
- vmx_inject_gp(mach, vcpu);
+ vmx_inject_gp(vcpu);
return true;
}
@@ -1642,7 +1656,7 @@ vmx_exit_xsetbv(struct nvmm_machine *mac
return;
error:
- vmx_inject_gp(mach, vcpu);
+ vmx_inject_gp(vcpu);
}
#define VMX_EPT_VIOLATION_READ __BIT(0)
@@ -1873,6 +1887,10 @@ vmx_vcpu_run(struct nvmm_machine *mach,
vmx_vmcs_enter(vcpu);
+ if (__predict_false(vmx_vcpu_event_commit(vcpu) != 0)) {
+ vmx_vmcs_leave(vcpu);
+ return EINVAL;
+ }
vmx_vcpu_state_commit(vcpu);
comm->state_cached = 0;
@@ -1984,7 +2002,7 @@ vmx_vcpu_run(struct nvmm_machine *mach,
case VMCS_EXITCODE_VMWRITE:
case VMCS_EXITCODE_VMXOFF:
case VMCS_EXITCODE_VMXON:
- vmx_inject_ud(mach, vcpu);
+ vmx_inject_ud(vcpu);
exit->reason = NVMM_EXIT_NONE;
break;
case VMCS_EXITCODE_EPT_VIOLATION:
