bhyve VMs are suspended using the "--suspend" argument to the "bhyvectl" tool:
bhyvectl --suspend=<snapshot_path> --vm=<vm_name> --rundir=<rundir> Where <snapshot_path> is the base filename for the snapshot files, <vm_name> is a VM name, and <rundir> points to a directory with the control socket. Resume uses the same "bhyve" command as a normal VM startup, but adds "-r" with a path to the <snapshot_path>. For non-UEFI VMs, it also requires running "bhyveload". Add the bhyveDomainSuspend() which implements domain suspend. Add bhyveDomainResume() which implements domain resume, using virBhyveProcessResume() as a wrapper to virBhyveProcessStartImpl(). virBhyveProcessStartImpl() now accepts the "resume" argument which controls whether the "-r" resume argument should be added. Signed-off-by: Roman Bogorodskiy <[email protected]> --- src/bhyve/bhyve_command.c | 22 +++++++- src/bhyve/bhyve_command.h | 5 ++ src/bhyve/bhyve_conf.c | 3 ++ src/bhyve/bhyve_driver.c | 102 +++++++++++++++++++++++++++++++++++++- src/bhyve/bhyve_process.c | 68 ++++++++++++++++++++----- src/bhyve/bhyve_process.h | 6 +++ src/bhyve/bhyve_utils.h | 1 + tests/bhyvexml2argvtest.c | 2 +- 8 files changed, 193 insertions(+), 16 deletions(-) diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c index be66d26c50..f1a6d9878f 100644 --- a/src/bhyve/bhyve_command.c +++ b/src/bhyve/bhyve_command.c @@ -1147,7 +1147,7 @@ bhyveBuildSysinfoArgStr(virSysinfoDef *sysinfo, virCommand * virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, - bool dryRun) + bool resume, bool dryRun) { /* * /usr/sbin/bhyve -c 2 -m 256 -AI -H -P \ @@ -1164,6 +1164,12 @@ virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, unsigned nvcpus = virDomainDefGetVcpus(def); size_t ncells = virDomainNumaGetNodeCount(def->numa); + if (resume) { + virCommandAddArg(cmd, "-r"); + virCommandAddArgFormat(cmd, "%s/%s/snapshot", driver->config->snapshotDir, + def->name); + } + /* Monitor mode */ if (!driver->privileged) { virCommandAddArg(cmd, "-M"); @@ -1427,6 +1433,20 @@ virBhyveProcessBuildResetCmd(struct _bhyveConn *driver G_GNUC_UNUSED, return cmd; } +virCommand * +virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver, + virDomainDef *def) +{ + virCommand *cmd = virCommandNew("bhyvectl"); + + virCommandAddArgFormat(cmd, "--suspend=%s/%s/snapshot", + driver->config->snapshotDir, def->name); + virCommandAddArgPair(cmd, "--vm", def->name); + virCommandAddArgPair(cmd, "--rundir", driver->config->bhyveRunDir); + + return cmd; +} + virCommand * virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver, virDomainDef *def) diff --git a/src/bhyve/bhyve_command.h b/src/bhyve/bhyve_command.h index 2ee3f3a607..32a6b27ca3 100644 --- a/src/bhyve/bhyve_command.h +++ b/src/bhyve/bhyve_command.h @@ -31,6 +31,7 @@ virCommand *virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def, + bool resume, bool dryRun); virCommand * @@ -44,5 +45,9 @@ virBhyveProcessBuildLoadCmd(struct _bhyveConn *driver, virDomainDef *def, virCommand * virBhyveProcessBuildResetCmd(struct _bhyveConn *driver, virDomainDef *def); +virCommand * +virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver, + virDomainDef *def); + virCommand * virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver, virDomainDef *def); diff --git a/src/bhyve/bhyve_conf.c b/src/bhyve/bhyve_conf.c index 68bd22f00a..c6895fe245 100644 --- a/src/bhyve/bhyve_conf.c +++ b/src/bhyve/bhyve_conf.c @@ -94,6 +94,8 @@ virBhyveDriverConfigNew(bool privileged) cfg->logDir = g_strdup_printf("%s/bhyve/log", cachedir); } + cfg->snapshotDir = g_build_filename(cfg->stateDir, "snapshot", NULL); + cfg->bhyveloadTimeout = 300; cfg->bhyveloadTimeoutKill = 15; @@ -153,6 +155,7 @@ virBhyveDriverConfigDispose(void *obj) g_free(cfg->logDir); g_free(cfg->nvramDir); g_free(cfg->stateDir); + g_free(cfg->snapshotDir); g_free(cfg->bhyveRunDir); g_free(cfg->ubootPath); diff --git a/src/bhyve/bhyve_driver.c b/src/bhyve/bhyve_driver.c index 47ed7d6963..bae1961036 100644 --- a/src/bhyve/bhyve_driver.c +++ b/src/bhyve/bhyve_driver.c @@ -797,7 +797,7 @@ bhyveConnectDomainXMLToNative(virConnectPtr conn, virBufferAddChar(&buf, '\n'); } - if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, true))) + if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, false, true))) return NULL; virCommandToStringBuf(cmd, &buf, false, false); @@ -1475,6 +1475,13 @@ bhyveStateInitialize(bool privileged, goto cleanup; } + if (g_mkdir_with_parents(cfg->snapshotDir, 0700) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %1$s"), + cfg->snapshotDir); + goto cleanup; + } + if (g_mkdir_with_parents(cfg->nvramDir, 0777) < 0) { virReportSystemError(errno, _("Failed to mkdir %1$s"), @@ -3226,6 +3233,97 @@ bhyveDomainGetGuestInfo(virDomainPtr domain, } +static int +bhyveDomainSuspend(virDomainPtr domain) +{ + struct _bhyveConn *driver = domain->conn->privateData; + g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver); + virDomainObj *vm; + virObjectEvent *event = NULL; + int ret = -1; + + if (!(vm = bhyveDomObjFromDomain(domain))) + return -1; + + if (virDomainSuspendEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), BHYVE_CAP_SUSPEND_RESUME)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("bhyve does not support suspend/resume")); + goto cleanup; + } + + if (virDomainObjCheckActive(vm) < 0) + goto cleanup; + + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) { + if (virBhyveProcessSuspend(driver, vm) < 0) + goto cleanup; + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_SUSPENDED, + VIR_DOMAIN_EVENT_SUSPENDED_PAUSED); + } + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virObjectEventStateQueue(driver->domainEventState, event); + virDomainObjEndAPI(&vm); + return ret; +} + + +static int +bhyveDomainResume(virDomainPtr domain) +{ + struct _bhyveConn *driver = domain->conn->privateData; + g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver); + virDomainObj *vm; + virObjectEvent *event = NULL; + int ret = -1; + + if (!(vm = bhyveDomObjFromDomain(domain))) + return -1; + + if (virDomainResumeEnsureACL(domain->conn, vm->def) < 0) + goto cleanup; + + if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), BHYVE_CAP_SUSPEND_RESUME)) { + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s", + _("bhyve does not support suspend/resume")); + goto cleanup; + } + + if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) { + virReportError(VIR_ERR_OPERATION_INVALID, "%s", + _("cannot resume domain not in paused state")); + goto cleanup; + } + + if (virBhyveProcessResume(driver, vm) < 0) + goto cleanup; + + event = virDomainEventLifecycleNewFromObj(vm, + VIR_DOMAIN_EVENT_RESUMED, + VIR_DOMAIN_EVENT_RESUMED_UNPAUSED); + + if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0) + goto cleanup; + + ret = 0; + + cleanup: + virObjectEventStateQueue(driver->domainEventState, event); + virDomainObjEndAPI(&vm); + return ret; +} + + static int bhyveDomainReset(virDomainPtr domain, unsigned int flags) { @@ -3334,6 +3432,8 @@ static virHypervisorDriver bhyveHypervisorDriver = { .domainGetGuestInfo = bhyveDomainGetGuestInfo, /* 12.7.0 */ .domainSetLifecycleAction = bhyveDomainSetLifecycleAction, /* 12.8.0 */ .domainReset = bhyveDomainReset, /* 12.9.0 */ + .domainSuspend = bhyveDomainSuspend, /* 12.10.0 */ + .domainResume = bhyveDomainResume, /* 12.10.0 */ }; diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c index c27a5cf6ed..0cac9e841a 100644 --- a/src/bhyve/bhyve_process.c +++ b/src/bhyve/bhyve_process.c @@ -377,7 +377,8 @@ bhyveConnectAgent(struct _bhyveConn *driver G_GNUC_UNUSED, virDomainObj *vm) static int virBhyveProcessStartImpl(struct _bhyveConn *driver, virDomainObj *vm, - virDomainRunningReason reason) + virDomainRunningReason reason, + bool resume) { g_autofree char *devmap_file = NULL; g_autofree char *devicemap = NULL; @@ -421,7 +422,7 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, goto cleanup; /* Call bhyve to start the VM */ - if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, false))) + if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, resume, false))) goto cleanup; virCommandSetOutputFD(cmd, &logfd); @@ -493,8 +494,11 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver, goto cleanup; } - /* /dev/vmm can appear before the monitor mode IPC socket is ready. */ - if (!driver->privileged && bhyveProcessWaitForVMPid(vm) < 0) + /* /dev/vmm can appear before the monitor mode IPC socket is ready. + * A loader also creates it before bhyve starts restoring + * a suspended VM. */ + if ((!driver->privileged || resume) && + bhyveProcessWaitForVMPid(vm) < 0) goto cleanup; vm->def->id = vm->pid; @@ -698,7 +702,7 @@ virBhyveProcessStart(bhyveConn *driver, if (bhyveProcessPrepareHost(driver, vm->def, flags) < 0) goto cleanup; - return virBhyveProcessStartImpl(driver, vm, reason); + return virBhyveProcessStartImpl(driver, vm, reason, false); cleanup: virErrorPreserveLast(&save_err); @@ -756,9 +760,10 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver, static int virBhyveProcessStopImpl(struct _bhyveConn *driver, virDomainObj *vm, - virDomainShutoffReason reason, + int reason, bool forceCleanup, - bool restoreDef) + bool restoreDef, + bool suspend) { int ret = 0; size_t i = 0; @@ -787,6 +792,20 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, } } + if (suspend) { + g_autofree char *snapshot_dir = g_build_filename(driver->config->snapshotDir, + vm->def->name, NULL); + + if (g_mkdir_with_parents(snapshot_dir, 0700) < 0) { + virReportSystemError(errno, + _("Failed to mkdir %1$s"), snapshot_dir); + ret = -1; + if (!save_err) + virErrorPreserveLast(&save_err); + goto cleanup; + } + } + /* Destroy monitor before running the actual destroy command to prevent * it from detecting VM shutdown and entering this cleanup routine again */ if ((priv != NULL) && (priv->mon != NULL)) @@ -794,7 +813,10 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virBhyveDomainObjStopWorker(vm); - cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); + if (suspend) + cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def); + else + cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def); if (virCommandRun(cmd, NULL) < 0) { /* Only failure of the actual destroy command warrants unsuccessful return code, * other failures are not considered critical */ @@ -832,7 +854,9 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy); - virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason); + virDomainObjSetState(vm, + suspend ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF, + reason); vm->pid = 0; vm->def->id = -1; @@ -842,7 +866,7 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver, bhyveProcessRemoveDomainStatus(driver->config->stateDir, vm->def->name); bhyveProcessRemoveIPCSocket(driver, vm->def->name); - if (restoreDef) + if (restoreDef && !suspend) virDomainObjRemoveTransientDef(vm); cleanup: @@ -856,7 +880,16 @@ virBhyveProcessStop(struct _bhyveConn *driver, virDomainShutoffReason reason, bool forceCleanup) { - return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true); + return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true, + false); +} + +int +virBhyveProcessSuspend(struct _bhyveConn *driver, + virDomainObj *vm) +{ + return virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_PAUSED_USER, + false, false, true); } int @@ -885,10 +918,11 @@ virBhyveProcessRestart(struct _bhyveConn *driver, virDomainObj *vm) { if (virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN, - false, false) < 0) + false, false, false) < 0) return -1; - if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED) < 0) + if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED, + false) < 0) return -1; return 0; @@ -905,6 +939,14 @@ virBhyveProcessReset(struct _bhyveConn *driver, virDomainObj *vm) return 0; } +int +virBhyveProcessResume(struct _bhyveConn *driver, + virDomainObj *vm) +{ + return virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED, + true); +} + int virBhyveGetDomainTotalCpuStats(virDomainObj *vm, unsigned long long *cpustats) diff --git a/src/bhyve/bhyve_process.h b/src/bhyve/bhyve_process.h index 2f0ce21167..655121a150 100644 --- a/src/bhyve/bhyve_process.h +++ b/src/bhyve/bhyve_process.h @@ -47,12 +47,18 @@ int virBhyveProcessStop(struct _bhyveConn *driver, virDomainShutoffReason reason, bool forceCleanup); +int virBhyveProcessSuspend(struct _bhyveConn *driver, + virDomainObj *vm); + int virBhyveProcessRestart(struct _bhyveConn *driver, virDomainObj *vm); int virBhyveProcessReset(struct _bhyveConn *driver, virDomainObj *vm); +int virBhyveProcessResume(struct _bhyveConn *driver, + virDomainObj *vm); + int virBhyveProcessShutdown(virDomainObj *vm); pid_t virBhyveProcessGetBhyvePid(virDomainObj *vm); diff --git a/src/bhyve/bhyve_utils.h b/src/bhyve/bhyve_utils.h index 6d15035693..fce67dfdf3 100644 --- a/src/bhyve/bhyve_utils.h +++ b/src/bhyve/bhyve_utils.h @@ -42,6 +42,7 @@ struct _virBhyveDriverConfig { char *logDir; char *nvramDir; char *stateDir; + char *snapshotDir; char *bhyveRunDir; char *ubootPath; diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c index f5df8a0265..078d5faae1 100644 --- a/tests/bhyvexml2argvtest.c +++ b/tests/bhyvexml2argvtest.c @@ -70,7 +70,7 @@ static int testCompareXMLToArgvFiles(const char *xml, goto out; } - cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false); + cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false, false); if (vm->def->os.loader) ldcmd = virCommandNew("dummy"); else -- 2.56.0
