Module Name: src
Committed By: martin
Date: Sat Aug 29 17:00:28 UTC 2020
Modified Files:
src/sys/dev/nvmm [netbsd-9]: nvmm.c nvmm_ioctl.h
src/sys/dev/nvmm/x86 [netbsd-9]: nvmm_x86.c nvmm_x86_svm.c
nvmm_x86_svmfunc.S nvmm_x86_vmx.c nvmm_x86_vmxfunc.S
Log Message:
Pull up following revision(s) (requested by maxv in ticket #1068):
sys/dev/nvmm/x86/nvmm_x86_svm.c: revision 1.71
sys/dev/nvmm/nvmm.c: revision 1.34
sys/dev/nvmm/x86/nvmm_x86_svm.c: revision 1.72
sys/dev/nvmm/nvmm.c: revision 1.35
sys/dev/nvmm/nvmm.c: revision 1.36
sys/dev/nvmm/x86/nvmm_x86_svmfunc.S: revision 1.5
sys/dev/nvmm/nvmm.c: revision 1.37
sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S: revision 1.5
sys/dev/nvmm/x86/nvmm_x86_vmx.c: revision 1.70
sys/dev/nvmm/x86/nvmm_x86_svm.c: revision 1.68
sys/dev/nvmm/x86/nvmm_x86.c: revision 1.15
sys/dev/nvmm/nvmm_ioctl.h: revision 1.10
Micro-optimize: use pushq instead of pushw. To avoid LCP stalls and
unaligned stack accesses.
nvmm-x86: also flush the guest TLB when CR4.{PCIDE,SMEP} changes
nvmm: localify a variable that doesn't need to be global
nvmm: use relaxed atomics to read nmachines
nvmm-x86-svm: dedup code
nvmm-x86: hide more CPUID flags, mostly related to perf monitors
nvmm: misc improvements
- use mach->ncpus to get the number of vcpus, now that we have it
- don't forget to decrement mach->ncpus when a machine gets killed
- add more __predict_false()
nvmm-x86-svm: don't forget to intercept INVD
INVD executed in the guest can be dangerous for the host, due to CPU
caches being flushed without write-back.
nvmm: slightly clarify
nvmm: explicitly include atomic.h
To generate a diff of this commit:
cvs rdiff -u -r1.22.2.6 -r1.22.2.7 src/sys/dev/nvmm/nvmm.c
cvs rdiff -u -r1.7.4.1 -r1.7.4.2 src/sys/dev/nvmm/nvmm_ioctl.h
cvs rdiff -u -r1.7.4.4 -r1.7.4.5 src/sys/dev/nvmm/x86/nvmm_x86.c
cvs rdiff -u -r1.46.4.9 -r1.46.4.10 src/sys/dev/nvmm/x86/nvmm_x86_svm.c
cvs rdiff -u -r1.3 -r1.3.4.1 src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S \
src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S
cvs rdiff -u -r1.36.2.11 -r1.36.2.12 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/nvmm/nvmm.c
diff -u src/sys/dev/nvmm/nvmm.c:1.22.2.6 src/sys/dev/nvmm/nvmm.c:1.22.2.7
--- src/sys/dev/nvmm/nvmm.c:1.22.2.6 Tue Aug 18 09:29:52 2020
+++ src/sys/dev/nvmm/nvmm.c Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm.c,v 1.22.2.6 2020/08/18 09:29:52 martin Exp $ */
+/* $NetBSD: nvmm.c,v 1.22.2.7 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
@@ -30,12 +30,13 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.22.2.6 2020/08/18 09:29:52 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm.c,v 1.22.2.7 2020/08/29 17:00:28 martin Exp $");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/atomic.h>
#include <sys/cpu.h>
#include <sys/conf.h>
#include <sys/kmem.h>
@@ -112,17 +113,17 @@ nvmm_machine_get(struct nvmm_owner *owne
struct nvmm_machine *mach;
krw_t op = writer ? RW_WRITER : RW_READER;
- if (machid >= NVMM_MAX_MACHINES) {
+ if (__predict_false(machid >= NVMM_MAX_MACHINES)) {
return EINVAL;
}
mach = &machines[machid];
rw_enter(&mach->lock, op);
- if (!mach->present) {
+ if (__predict_false(!mach->present)) {
rw_exit(&mach->lock);
return ENOENT;
}
- if (owner != &root_owner && mach->owner != owner) {
+ if (__predict_false(mach->owner != owner && owner != &root_owner)) {
rw_exit(&mach->lock);
return EPERM;
}
@@ -179,13 +180,13 @@ nvmm_vcpu_get(struct nvmm_machine *mach,
{
struct nvmm_cpu *vcpu;
- if (cpuid >= NVMM_MAX_VCPUS) {
+ if (__predict_false(cpuid >= NVMM_MAX_VCPUS)) {
return EINVAL;
}
vcpu = &mach->cpus[cpuid];
mutex_enter(&vcpu->lock);
- if (!vcpu->present) {
+ if (__predict_false(!vcpu->present)) {
mutex_exit(&vcpu->lock);
return ENOENT;
}
@@ -227,6 +228,7 @@ nvmm_kill_machines(struct nvmm_owner *ow
(*nvmm_impl->vcpu_destroy)(mach, vcpu);
nvmm_vcpu_free(mach, vcpu);
nvmm_vcpu_put(vcpu);
+ atomic_dec_uint(&mach->ncpus);
}
(*nvmm_impl->machine_destroy)(mach);
uvmspace_free(mach->vm);
@@ -314,6 +316,7 @@ nvmm_machine_destroy(struct nvmm_owner *
(*nvmm_impl->vcpu_destroy)(mach, vcpu);
nvmm_vcpu_free(mach, vcpu);
nvmm_vcpu_put(vcpu);
+ atomic_dec_uint(&mach->ncpus);
}
(*nvmm_impl->machine_destroy)(mach);
@@ -414,7 +417,6 @@ nvmm_vcpu_create(struct nvmm_owner *owne
}
nvmm_vcpu_put(vcpu);
-
atomic_inc_uint(&mach->ncpus);
out:
@@ -440,7 +442,6 @@ nvmm_vcpu_destroy(struct nvmm_owner *own
(*nvmm_impl->vcpu_destroy)(mach, vcpu);
nvmm_vcpu_free(mach, vcpu);
nvmm_vcpu_put(vcpu);
-
atomic_dec_uint(&mach->ncpus);
out:
@@ -907,7 +908,6 @@ nvmm_ctl_mach_info(struct nvmm_owner *ow
{
struct nvmm_ctl_mach_info ctl;
struct nvmm_machine *mach;
- struct nvmm_cpu *vcpu;
int error;
size_t i;
@@ -921,14 +921,7 @@ nvmm_ctl_mach_info(struct nvmm_owner *ow
if (error)
return error;
- ctl.nvcpus = 0;
- for (i = 0; i < NVMM_MAX_VCPUS; i++) {
- error = nvmm_vcpu_get(mach, i, &vcpu);
- if (error)
- continue;
- ctl.nvcpus++;
- nvmm_vcpu_put(vcpu);
- }
+ ctl.nvcpus = mach->ncpus;
ctl.nram = 0;
for (i = 0; i < NVMM_MAX_HMAPPINGS; i++) {
@@ -1040,7 +1033,7 @@ static int nvmm_close(file_t *);
static int nvmm_mmap(file_t *, off_t *, size_t, int, int *, int *,
struct uvm_object **, int *);
-const struct fileops nvmm_fileops = {
+static const struct fileops nvmm_fileops = {
.fo_read = fbadop_read,
.fo_write = fbadop_write,
.fo_ioctl = nvmm_ioctl,
@@ -1218,7 +1211,7 @@ nvmm_attach(device_t parent, device_t se
static int
nvmm_detach(device_t self, int flags)
{
- if (nmachines > 0)
+ if (atomic_load_relaxed(&nmachines) > 0)
return EBUSY;
nvmm_fini();
return 0;
Index: src/sys/dev/nvmm/nvmm_ioctl.h
diff -u src/sys/dev/nvmm/nvmm_ioctl.h:1.7.4.1 src/sys/dev/nvmm/nvmm_ioctl.h:1.7.4.2
--- src/sys/dev/nvmm/nvmm_ioctl.h:1.7.4.1 Sun Nov 10 12:58:30 2019
+++ src/sys/dev/nvmm/nvmm_ioctl.h Sat Aug 29 17:00:28 2020
@@ -1,7 +1,7 @@
-/* $NetBSD: nvmm_ioctl.h,v 1.7.4.1 2019/11/10 12:58:30 martin Exp $ */
+/* $NetBSD: nvmm_ioctl.h,v 1.7.4.2 2020/08/29 17:00:28 martin Exp $ */
/*
- * Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
+ * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
@@ -121,7 +121,9 @@ struct nvmm_ioc_gpa_unmap {
};
struct nvmm_ctl_mach_info {
+ /* input */
nvmm_machid_t machid;
+ /* output */
uint32_t nvcpus;
uint64_t nram;
pid_t pid;
@@ -151,7 +153,6 @@ struct nvmm_ioc_ctl {
#define NVMM_IOC_GPA_UNMAP _IOW ('N', 12, struct nvmm_ioc_gpa_unmap)
#define NVMM_IOC_HVA_MAP _IOW ('N', 13, struct nvmm_ioc_hva_map)
#define NVMM_IOC_HVA_UNMAP _IOW ('N', 14, struct nvmm_ioc_hva_unmap)
-
#define NVMM_IOC_CTL _IOW ('N', 20, struct nvmm_ioc_ctl)
#endif /* _NVMM_IOCTL_H_ */
Index: src/sys/dev/nvmm/x86/nvmm_x86.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86.c:1.7.4.4 src/sys/dev/nvmm/x86/nvmm_x86.c:1.7.4.5
--- src/sys/dev/nvmm/x86/nvmm_x86.c:1.7.4.4 Wed Aug 26 17:55:49 2020
+++ src/sys/dev/nvmm/x86/nvmm_x86.c Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86.c,v 1.7.4.4 2020/08/26 17:55:49 martin Exp $ */
+/* $NetBSD: nvmm_x86.c,v 1.7.4.5 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86.c,v 1.7.4.4 2020/08/26 17:55:49 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86.c,v 1.7.4.5 2020/08/29 17:00:28 martin Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -389,22 +389,22 @@ const struct nvmm_x86_cpuid_mask nvmm_cp
CPUID_MISALIGNSSE |
CPUID_3DNOWPF |
/* CPUID_OSVW excluded */
- CPUID_IBS |
+ /* CPUID_IBS excluded */
CPUID_XOP |
/* CPUID_SKINIT excluded */
- CPUID_WDT |
- CPUID_LWP |
+ /* CPUID_WDT excluded */
+ /* CPUID_LWP excluded */
CPUID_FMA4 |
CPUID_TCE |
- CPUID_NODEID |
+ /* CPUID_NODEID excluded */
CPUID_TBM |
- CPUID_TOPOEXT |
- CPUID_PCEC |
- CPUID_PCENB |
- CPUID_SPM |
- CPUID_DBE |
- CPUID_PTSC |
- CPUID_L2IPERFC,
+ CPUID_TOPOEXT,
+ /* CPUID_PCEC excluded */
+ /* CPUID_PCENB excluded */
+ /* CPUID_SPM excluded */
+ /* CPUID_DBE excluded */
+ /* CPUID_PTSC excluded */
+ /* CPUID_L2IPERFC excluded */
/* CPUID_MWAITX excluded */
.edx =
CPUID_SYSCALL |
Index: src/sys/dev/nvmm/x86/nvmm_x86_svm.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.46.4.9 src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.46.4.10
--- src/sys/dev/nvmm/x86/nvmm_x86_svm.c:1.46.4.9 Wed Aug 26 17:55:48 2020
+++ src/sys/dev/nvmm/x86/nvmm_x86_svm.c Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_svm.c,v 1.46.4.9 2020/08/26 17:55:48 martin Exp $ */
+/* $NetBSD: nvmm_x86_svm.c,v 1.46.4.10 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.9 2020/08/26 17:55:48 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.10 2020/08/29 17:00:28 martin Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -509,7 +509,7 @@ static uint64_t svm_xcr0_mask __read_mos
#define CR0_TLB_FLUSH \
(CR0_PG|CR0_WP|CR0_CD|CR0_NW)
#define CR4_TLB_FLUSH \
- (CR4_PGE|CR4_PAE|CR4_PSE)
+ (CR4_PSE|CR4_PAE|CR4_PGE|CR4_PCIDE|CR4_SMEP)
/* -------------------------------------------------------------------------- */
@@ -1012,18 +1012,11 @@ svm_exit_cpuid(struct nvmm_machine *mach
struct svm_cpudata *cpudata = vcpu->cpudata;
struct nvmm_vcpu_conf_cpuid *cpuid;
uint64_t eax, ecx;
- u_int descs[4];
size_t i;
eax = cpudata->vmcb->state.rax;
ecx = cpudata->gprs[NVMM_X64_GPR_RCX];
- x86_cpuid2(eax, ecx, descs);
-
- cpudata->vmcb->state.rax = descs[0];
- cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1];
- cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2];
- cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3];
-
+ svm_inkernel_exec_cpuid(cpudata, eax, ecx);
svm_inkernel_handle_cpuid(vcpu, eax, ecx);
for (i = 0; i < SVM_NCPUIDS; i++) {
@@ -2118,7 +2111,6 @@ svm_vcpu_init(struct nvmm_machine *mach,
* - POPF [popf instruction]
* - IRET [iret instruction]
* - INTN [int $n instructions]
- * - INVD [invd instruction]
* - PAUSE [pause instruction]
* - INVLPG [invplg instruction]
* - TASKSW [task switches]
@@ -2132,6 +2124,7 @@ svm_vcpu_init(struct nvmm_machine *mach,
VMCB_CTRL_INTERCEPT_RDPMC |
VMCB_CTRL_INTERCEPT_CPUID |
VMCB_CTRL_INTERCEPT_RSM |
+ VMCB_CTRL_INTERCEPT_INVD |
VMCB_CTRL_INTERCEPT_HLT |
VMCB_CTRL_INTERCEPT_INVLPGA |
VMCB_CTRL_INTERCEPT_IOIO_PROT |
Index: src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S
diff -u src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S:1.3 src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S:1.3.4.1
--- src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S:1.3 Wed Apr 24 18:45:15 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_svmfunc.S Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_svmfunc.S,v 1.3 2019/04/24 18:45:15 maxv Exp $ */
+/* $NetBSD: nvmm_x86_svmfunc.S,v 1.3.4.1 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -75,10 +75,10 @@
#define HOST_SAVE_TR \
strw %ax ;\
- pushw %ax
+ pushq %rax
#define HOST_RESTORE_TR \
- popw %ax ;\
+ popq %rax ;\
movzwq %ax,%rdx ;\
movq CPUVAR(GDT),%rax ;\
andq $~0x0200,4(%rax,%rdx, 1) ;\
@@ -86,10 +86,10 @@
#define HOST_SAVE_LDT \
sldtw %ax ;\
- pushw %ax
+ pushq %rax
#define HOST_RESTORE_LDT \
- popw %ax ;\
+ popq %rax ;\
lldtw %ax
/*
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S:1.3 src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S:1.3.4.1
--- src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S:1.3 Sat Apr 27 08:16:19 2019
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmxfunc.S Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmxfunc.S,v 1.3 2019/04/27 08:16:19 maxv Exp $ */
+/* $NetBSD: nvmm_x86_vmxfunc.S,v 1.3.4.1 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018 The NetBSD Foundation, Inc.
@@ -100,10 +100,10 @@ END(_vmx_vmxoff)
#define HOST_SAVE_LDT \
sldtw %ax ;\
- pushw %ax
+ pushq %rax
#define HOST_RESTORE_LDT \
- popw %ax ;\
+ popq %rax ;\
lldtw %ax
/*
Index: src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -u src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.36.2.11 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.36.2.12
--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c:1.36.2.11 Wed Aug 26 17:55:49 2020
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c Sat Aug 29 17:00:28 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: nvmm_x86_vmx.c,v 1.36.2.11 2020/08/26 17:55:49 martin Exp $ */
+/* $NetBSD: nvmm_x86_vmx.c,v 1.36.2.12 2020/08/29 17:00:28 martin Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.11 2020/08/26 17:55:49 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.12 2020/08/29 17:00:28 martin Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -722,7 +722,7 @@ static uint64_t vmx_xcr0_mask __read_mos
#define CR0_TLB_FLUSH \
(CR0_PG|CR0_WP|CR0_CD|CR0_NW)
#define CR4_TLB_FLUSH \
- (CR4_PGE|CR4_PAE|CR4_PSE)
+ (CR4_PSE|CR4_PAE|CR4_PGE|CR4_PCIDE|CR4_SMEP)
/* -------------------------------------------------------------------------- */
