Author: grehan
Date: Sun Aug 17 00:52:07 2014
New Revision: 270070
URL: http://svnweb.freebsd.org/changeset/base/270070
Log:
MFC r266933
Activate vcpus from bhyve(8) using the ioctl VM_ACTIVATE_CPU instead of doing
it implicitly in vmm.ko.
Modified:
stable/10/lib/libvmmapi/vmmapi.c
stable/10/lib/libvmmapi/vmmapi.h
stable/10/sys/amd64/include/vmm.h
stable/10/sys/amd64/include/vmm_dev.h
stable/10/sys/amd64/vmm/io/vlapic.c
stable/10/sys/amd64/vmm/vmm.c
stable/10/sys/amd64/vmm/vmm_dev.c
stable/10/usr.sbin/bhyve/bhyverun.c
stable/10/usr.sbin/bhyve/pci_lpc.c
stable/10/usr.sbin/bhyvectl/bhyvectl.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/lib/libvmmapi/vmmapi.c
==============================================================================
--- stable/10/lib/libvmmapi/vmmapi.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/lib/libvmmapi/vmmapi.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -29,11 +29,12 @@
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
-#include <sys/types.h>
+#include <sys/param.h>
#include <sys/sysctl.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/_iovec.h>
+#include <sys/cpuset.h>
#include <machine/specialreg.h>
#include <machine/param.h>
@@ -1043,3 +1044,44 @@ vm_copyout(struct vmctx *ctx, int vcpu,
len -= n;
}
}
+
+static int
+vm_get_cpus(struct vmctx *ctx, int which, cpuset_t *cpus)
+{
+ struct vm_cpuset vm_cpuset;
+ int error;
+
+ bzero(&vm_cpuset, sizeof(struct vm_cpuset));
+ vm_cpuset.which = which;
+ vm_cpuset.cpusetsize = sizeof(cpuset_t);
+ vm_cpuset.cpus = cpus;
+
+ error = ioctl(ctx->fd, VM_GET_CPUS, &vm_cpuset);
+ return (error);
+}
+
+int
+vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus)
+{
+
+ return (vm_get_cpus(ctx, VM_ACTIVE_CPUS, cpus));
+}
+
+int
+vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus)
+{
+
+ return (vm_get_cpus(ctx, VM_SUSPENDED_CPUS, cpus));
+}
+
+int
+vm_activate_cpu(struct vmctx *ctx, int vcpu)
+{
+ struct vm_activate_cpu ac;
+ int error;
+
+ bzero(&ac, sizeof(struct vm_activate_cpu));
+ ac.vcpuid = vcpu;
+ error = ioctl(ctx->fd, VM_ACTIVATE_CPU, &ac);
+ return (error);
+}
Modified: stable/10/lib/libvmmapi/vmmapi.h
==============================================================================
--- stable/10/lib/libvmmapi/vmmapi.h Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/lib/libvmmapi/vmmapi.h Sun Aug 17 00:52:07 2014
(r270070)
@@ -29,6 +29,9 @@
#ifndef _VMMAPI_H_
#define _VMMAPI_H_
+#include <sys/param.h>
+#include <sys/cpuset.h>
+
struct iovec;
struct vmctx;
enum x2apic_state;
@@ -125,6 +128,10 @@ void vm_copyout(struct vmctx *ctx, int v
/* Reset vcpu register state */
int vcpu_reset(struct vmctx *ctx, int vcpu);
+int vm_active_cpus(struct vmctx *ctx, cpuset_t *cpus);
+int vm_suspended_cpus(struct vmctx *ctx, cpuset_t *cpus);
+int vm_activate_cpu(struct vmctx *ctx, int vcpu);
+
/*
* FreeBSD specific APIs
*/
Modified: stable/10/sys/amd64/include/vmm.h
==============================================================================
--- stable/10/sys/amd64/include/vmm.h Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/sys/amd64/include/vmm.h Sun Aug 17 00:52:07 2014
(r270070)
@@ -140,8 +140,9 @@ int vm_set_capability(struct vm *vm, int
int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state);
int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state);
int vm_apicid2vcpuid(struct vm *vm, int apicid);
-void vm_activate_cpu(struct vm *vm, int vcpu);
+int vm_activate_cpu(struct vm *vm, int vcpu);
cpuset_t vm_active_cpus(struct vm *vm);
+cpuset_t vm_suspended_cpus(struct vm *vm);
struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
Modified: stable/10/sys/amd64/include/vmm_dev.h
==============================================================================
--- stable/10/sys/amd64/include/vmm_dev.h Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/sys/amd64/include/vmm_dev.h Sun Aug 17 00:52:07 2014
(r270070)
@@ -177,6 +177,18 @@ struct vm_gla2gpa {
uint64_t gpa;
};
+struct vm_activate_cpu {
+ int vcpuid;
+};
+
+struct vm_cpuset {
+ int which;
+ int cpusetsize;
+ cpuset_t *cpus;
+};
+#define VM_ACTIVE_CPUS 0
+#define VM_SUSPENDED_CPUS 1
+
enum {
/* general routines */
IOCNUM_ABIVERS = 0,
@@ -229,6 +241,10 @@ enum {
IOCNUM_ISA_DEASSERT_IRQ = 81,
IOCNUM_ISA_PULSE_IRQ = 82,
IOCNUM_ISA_SET_IRQ_TRIGGER = 83,
+
+ /* vm_cpuset */
+ IOCNUM_ACTIVATE_CPU = 90,
+ IOCNUM_GET_CPUSET = 91,
};
#define VM_RUN \
@@ -301,4 +317,8 @@ enum {
_IOWR('v', IOCNUM_GET_GPA_PMAP, struct vm_gpa_pte)
#define VM_GLA2GPA \
_IOWR('v', IOCNUM_GLA2GPA, struct vm_gla2gpa)
+#define VM_ACTIVATE_CPU \
+ _IOW('v', IOCNUM_ACTIVATE_CPU, struct vm_activate_cpu)
+#define VM_GET_CPUS \
+ _IOW('v', IOCNUM_GET_CPUSET, struct vm_cpuset)
#endif
Modified: stable/10/sys/amd64/vmm/io/vlapic.c
==============================================================================
--- stable/10/sys/amd64/vmm/io/vlapic.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/sys/amd64/vmm/io/vlapic.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -1004,11 +1004,7 @@ vlapic_icrlo_write_handler(struct vlapic
if (vlapic2->boot_state != BS_SIPI)
return (0);
- /*
- * XXX this assumes that the startup IPI always succeeds
- */
vlapic2->boot_state = BS_RUNNING;
- vm_activate_cpu(vlapic2->vm, dest);
*retu = true;
vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
Modified: stable/10/sys/amd64/vmm/vmm.c
==============================================================================
--- stable/10/sys/amd64/vmm/vmm.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/sys/amd64/vmm/vmm.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -342,8 +342,6 @@ vm_create(const char *name, struct vm **
struct vm *vm;
struct vmspace *vmspace;
- const int BSP = 0;
-
/*
* If vmm.ko could not be successfully initialized then don't attempt
* to create the virtual machine.
@@ -373,8 +371,6 @@ vm_create(const char *name, struct vm **
guest_msrs_init(vm, i);
}
- vm_activate_cpu(vm, BSP);
-
*retvm = vm;
return (0);
}
@@ -1294,6 +1290,12 @@ vm_run(struct vm *vm, struct vm_run *vmr
if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
return (EINVAL);
+ if (!CPU_ISSET(vcpuid, &vm->active_cpus))
+ return (EINVAL);
+
+ if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
+ return (EINVAL);
+
rptr = &vm->rendezvous_func;
sptr = &vm->suspend;
pmap = vmspace_pmap(vm->vmspace);
@@ -1708,17 +1710,19 @@ vcpu_get_state(struct vm *vm, int vcpuid
return (state);
}
-void
+int
vm_activate_cpu(struct vm *vm, int vcpuid)
{
- KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU,
- ("vm_activate_cpu: invalid vcpuid %d", vcpuid));
- KASSERT(!CPU_ISSET(vcpuid, &vm->active_cpus),
- ("vm_activate_cpu: vcpuid %d is already active", vcpuid));
+ if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
+ return (EINVAL);
+
+ if (CPU_ISSET(vcpuid, &vm->active_cpus))
+ return (EBUSY);
VCPU_CTR0(vm, vcpuid, "activated");
CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
+ return (0);
}
cpuset_t
@@ -1728,6 +1732,13 @@ vm_active_cpus(struct vm *vm)
return (vm->active_cpus);
}
+cpuset_t
+vm_suspended_cpus(struct vm *vm)
+{
+
+ return (vm->suspended_cpus);
+}
+
void *
vcpu_stats(struct vm *vm, int vcpuid)
{
Modified: stable/10/sys/amd64/vmm/vmm_dev.c
==============================================================================
--- stable/10/sys/amd64/vmm/vmm_dev.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/sys/amd64/vmm/vmm_dev.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -146,7 +146,8 @@ static int
vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag,
struct thread *td)
{
- int error, vcpu, state_changed;
+ int error, vcpu, state_changed, size;
+ cpuset_t *cpuset;
struct vmmdev_softc *sc;
struct vm_memory_segment *seg;
struct vm_register *vmreg;
@@ -170,6 +171,8 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
struct vm_gpa_pte *gpapte;
struct vm_suspend *vmsuspend;
struct vm_gla2gpa *gg;
+ struct vm_activate_cpu *vac;
+ struct vm_cpuset *vm_cpuset;
sc = vmmdev_lookup2(cdev);
if (sc == NULL)
@@ -195,6 +198,7 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
case VM_PPTDEV_MSIX:
case VM_SET_X2APIC_STATE:
case VM_GLA2GPA:
+ case VM_ACTIVATE_CPU:
/*
* XXX fragile, handle with care
* Assumes that the first field of the ioctl data is the vcpu.
@@ -439,6 +443,29 @@ vmmdev_ioctl(struct cdev *cdev, u_long c
}
break;
}
+ case VM_ACTIVATE_CPU:
+ vac = (struct vm_activate_cpu *)data;
+ error = vm_activate_cpu(sc->vm, vac->vcpuid);
+ break;
+ case VM_GET_CPUS:
+ error = 0;
+ vm_cpuset = (struct vm_cpuset *)data;
+ size = vm_cpuset->cpusetsize;
+ if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) {
+ error = ERANGE;
+ break;
+ }
+ cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO);
+ if (vm_cpuset->which == VM_ACTIVE_CPUS)
+ *cpuset = vm_active_cpus(sc->vm);
+ else if (vm_cpuset->which == VM_SUSPENDED_CPUS)
+ *cpuset = vm_suspended_cpus(sc->vm);
+ else
+ error = EINVAL;
+ if (error == 0)
+ error = copyout(cpuset, vm_cpuset->cpus, size);
+ free(cpuset, M_TEMP);
+ break;
default:
error = ENOTTY;
break;
Modified: stable/10/usr.sbin/bhyve/bhyverun.c
==============================================================================
--- stable/10/usr.sbin/bhyve/bhyverun.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/usr.sbin/bhyve/bhyverun.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -242,6 +242,15 @@ fbsdrun_addcpu(struct vmctx *ctx, int fr
assert(fromcpu == BSP);
+ /*
+ * The 'newcpu' must be activated in the context of 'fromcpu'. If
+ * vm_activate_cpu() is delayed until newcpu's pthread starts running
+ * then vmm.ko is out-of-sync with bhyve and this can create a race
+ * with vm_suspend().
+ */
+ error = vm_activate_cpu(ctx, newcpu);
+ assert(error == 0);
+
CPU_SET_ATOMIC(newcpu, &cpumask);
/*
@@ -532,6 +541,7 @@ vm_loop(struct vmctx *ctx, int vcpu, uin
int error, rc, prevcpu;
enum vm_exitcode exitcode;
enum vm_suspend_how how;
+ cpuset_t active_cpus;
if (vcpumap[vcpu] != NULL) {
error = pthread_setaffinity_np(pthread_self(),
@@ -539,6 +549,9 @@ vm_loop(struct vmctx *ctx, int vcpu, uin
assert(error == 0);
}
+ error = vm_active_cpus(ctx, &active_cpus);
+ assert(CPU_ISSET(vcpu, &active_cpus));
+
while (1) {
error = vm_run(ctx, vcpu, rip, &vmexit[vcpu]);
if (error != 0)
Modified: stable/10/usr.sbin/bhyve/pci_lpc.c
==============================================================================
--- stable/10/usr.sbin/bhyve/pci_lpc.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/usr.sbin/bhyve/pci_lpc.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -32,7 +32,6 @@ __FBSDID("$FreeBSD$");
#include <sys/types.h>
#include <machine/vmm.h>
-#include <machine/vmm_dev.h>
#include <stdio.h>
#include <stdlib.h>
Modified: stable/10/usr.sbin/bhyvectl/bhyvectl.c
==============================================================================
--- stable/10/usr.sbin/bhyvectl/bhyvectl.c Sat Aug 16 22:55:58 2014
(r270069)
+++ stable/10/usr.sbin/bhyvectl/bhyvectl.c Sun Aug 17 00:52:07 2014
(r270070)
@@ -193,7 +193,9 @@ usage(void)
" [--assert-lapic-lvt=<pin>]\n"
" [--inject-nmi]\n"
" [--force-reset]\n"
- " [--force-poweroff]\n",
+ " [--force-poweroff]\n"
+ " [--get-active-cpus]\n"
+ " [--get-suspended-cpus]\n",
progname);
exit(1);
}
@@ -203,6 +205,7 @@ static int inject_nmi, assert_lapic_lvt;
static int force_reset, force_poweroff;
static const char *capname;
static int create, destroy, get_lowmem, get_highmem;
+static int get_active_cpus, get_suspended_cpus;
static uint64_t memsize;
static int set_cr0, get_cr0, set_cr3, get_cr3, set_cr4, get_cr4;
static int set_efer, get_efer;
@@ -390,6 +393,25 @@ enum {
ASSERT_LAPIC_LVT,
};
+static void
+print_cpus(const char *banner, const cpuset_t *cpus)
+{
+ int i, first;
+
+ first = 1;
+ printf("%s:\t", banner);
+ if (!CPU_EMPTY(cpus)) {
+ for (i = 0; i < CPU_SETSIZE; i++) {
+ if (CPU_ISSET(i, cpus)) {
+ printf("%s%d", first ? " " : ", ", i);
+ first = 0;
+ }
+ }
+ } else
+ printf(" (none)");
+ printf("\n");
+}
+
int
main(int argc, char *argv[])
{
@@ -401,6 +423,7 @@ main(int argc, char *argv[])
uint64_t ctl, eptp, bm, addr, u64, pteval[4], *pte;
struct vmctx *ctx;
int wired;
+ cpuset_t cpus;
uint64_t cr0, cr3, cr4, dr7, rsp, rip, rflags, efer, pat;
uint64_t rax, rbx, rcx, rdx, rsi, rdi, rbp;
@@ -570,6 +593,8 @@ main(int argc, char *argv[])
{ "inject-nmi", NO_ARG, &inject_nmi, 1 },
{ "force-reset", NO_ARG, &force_reset, 1 },
{ "force-poweroff", NO_ARG, &force_poweroff, 1 },
+ { "get-active-cpus", NO_ARG, &get_active_cpus, 1 },
+ { "get-suspended-cpus", NO_ARG, &get_suspended_cpus, 1 },
{ NULL, 0, NULL, 0 }
};
@@ -1529,6 +1554,18 @@ main(int argc, char *argv[])
}
}
+ if (!error && (get_active_cpus || get_all)) {
+ error = vm_active_cpus(ctx, &cpus);
+ if (!error)
+ print_cpus("active cpus", &cpus);
+ }
+
+ if (!error && (get_suspended_cpus || get_all)) {
+ error = vm_suspended_cpus(ctx, &cpus);
+ if (!error)
+ print_cpus("suspended cpus", &cpus);
+ }
+
if (!error && run) {
error = vm_get_register(ctx, vcpu, VM_REG_GUEST_RIP, &rip);
assert(error == 0);
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
