bhyve VMs are suspended using the "--suspend" argument
to the "bhyvectl" tool:

 bhyvectl --suspend=<snapshot_path> --vm=<vm_name> --rundir=<rundir>

Where <snapshot_path> is the base filename for the snapshot files,
<vm_name> is a VM name, and <rundir> points to a directory with the
control socket.

Resume uses the same "bhyve" command as a normal VM startup,
but adds "-r" with a path to the <snapshot_path>. For non-UEFI
VMs, it also requires running "bhyveload".

Add the bhyveDomainSuspend() which implements domain suspend.
Add bhyveDomainResume() which implements domain resume, using
virBhyveProcessResume() as a wrapper to virBhyveProcessStartImpl().

virBhyveProcessStartImpl() now accepts the "resume" argument
which controls whether the "-r" resume argument should be added.

Signed-off-by: Roman Bogorodskiy <[email protected]>
---
 src/bhyve/bhyve_command.c |  22 +++++++-
 src/bhyve/bhyve_command.h |   5 ++
 src/bhyve/bhyve_conf.c    |   3 ++
 src/bhyve/bhyve_driver.c  | 102 +++++++++++++++++++++++++++++++++++++-
 src/bhyve/bhyve_process.c |  68 ++++++++++++++++++++-----
 src/bhyve/bhyve_process.h |   6 +++
 src/bhyve/bhyve_utils.h   |   1 +
 tests/bhyvexml2argvtest.c |   2 +-
 8 files changed, 193 insertions(+), 16 deletions(-)

diff --git a/src/bhyve/bhyve_command.c b/src/bhyve/bhyve_command.c
index be66d26c50..f1a6d9878f 100644
--- a/src/bhyve/bhyve_command.c
+++ b/src/bhyve/bhyve_command.c
@@ -1147,7 +1147,7 @@ bhyveBuildSysinfoArgStr(virSysinfoDef *sysinfo,
 
 virCommand *
 virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def,
-                             bool dryRun)
+                             bool resume, bool dryRun)
 {
     /*
      * /usr/sbin/bhyve -c 2 -m 256 -AI -H -P \
@@ -1164,6 +1164,12 @@ virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, 
virDomainDef *def,
     unsigned nvcpus = virDomainDefGetVcpus(def);
     size_t ncells = virDomainNumaGetNodeCount(def->numa);
 
+    if (resume) {
+        virCommandAddArg(cmd, "-r");
+        virCommandAddArgFormat(cmd, "%s/%s/snapshot", 
driver->config->snapshotDir,
+                               def->name);
+    }
+
     /* Monitor mode */
     if (!driver->privileged) {
         virCommandAddArg(cmd, "-M");
@@ -1427,6 +1433,20 @@ virBhyveProcessBuildResetCmd(struct _bhyveConn *driver 
G_GNUC_UNUSED,
     return cmd;
 }
 
+virCommand *
+virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver,
+                               virDomainDef *def)
+{
+    virCommand *cmd = virCommandNew("bhyvectl");
+
+    virCommandAddArgFormat(cmd, "--suspend=%s/%s/snapshot",
+                           driver->config->snapshotDir, def->name);
+    virCommandAddArgPair(cmd, "--vm", def->name);
+    virCommandAddArgPair(cmd, "--rundir", driver->config->bhyveRunDir);
+
+    return cmd;
+}
+
 virCommand *
 virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver,
                                 virDomainDef *def)
diff --git a/src/bhyve/bhyve_command.h b/src/bhyve/bhyve_command.h
index 2ee3f3a607..32a6b27ca3 100644
--- a/src/bhyve/bhyve_command.h
+++ b/src/bhyve/bhyve_command.h
@@ -31,6 +31,7 @@
 
 virCommand *virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver,
                                          virDomainDef *def,
+                                         bool resume,
                                          bool dryRun);
 
 virCommand *
@@ -44,5 +45,9 @@ virBhyveProcessBuildLoadCmd(struct _bhyveConn *driver, 
virDomainDef *def,
 virCommand *
 virBhyveProcessBuildResetCmd(struct _bhyveConn *driver, virDomainDef *def);
 
+virCommand *
+virBhyveProcessBuildSuspendCmd(struct _bhyveConn *driver,
+                               virDomainDef *def);
+
 virCommand *
 virBhyveProcessBuildGetVMPidCmd(struct _bhyveConn *driver, virDomainDef *def);
diff --git a/src/bhyve/bhyve_conf.c b/src/bhyve/bhyve_conf.c
index 68bd22f00a..c6895fe245 100644
--- a/src/bhyve/bhyve_conf.c
+++ b/src/bhyve/bhyve_conf.c
@@ -94,6 +94,8 @@ virBhyveDriverConfigNew(bool privileged)
         cfg->logDir = g_strdup_printf("%s/bhyve/log", cachedir);
     }
 
+    cfg->snapshotDir = g_build_filename(cfg->stateDir, "snapshot", NULL);
+
     cfg->bhyveloadTimeout = 300;
     cfg->bhyveloadTimeoutKill = 15;
 
@@ -153,6 +155,7 @@ virBhyveDriverConfigDispose(void *obj)
     g_free(cfg->logDir);
     g_free(cfg->nvramDir);
     g_free(cfg->stateDir);
+    g_free(cfg->snapshotDir);
     g_free(cfg->bhyveRunDir);
 
     g_free(cfg->ubootPath);
diff --git a/src/bhyve/bhyve_driver.c b/src/bhyve/bhyve_driver.c
index 47ed7d6963..bae1961036 100644
--- a/src/bhyve/bhyve_driver.c
+++ b/src/bhyve/bhyve_driver.c
@@ -797,7 +797,7 @@ bhyveConnectDomainXMLToNative(virConnectPtr conn,
         virBufferAddChar(&buf, '\n');
     }
 
-    if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, true)))
+    if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, false, true)))
         return NULL;
 
     virCommandToStringBuf(cmd, &buf, false, false);
@@ -1475,6 +1475,13 @@ bhyveStateInitialize(bool privileged,
         goto cleanup;
     }
 
+    if (g_mkdir_with_parents(cfg->snapshotDir, 0700) < 0) {
+        virReportSystemError(errno,
+                             _("Failed to mkdir %1$s"),
+                             cfg->snapshotDir);
+        goto cleanup;
+    }
+
     if (g_mkdir_with_parents(cfg->nvramDir, 0777) < 0) {
         virReportSystemError(errno,
                              _("Failed to mkdir %1$s"),
@@ -3226,6 +3233,97 @@ bhyveDomainGetGuestInfo(virDomainPtr domain,
 }
 
 
+static int
+bhyveDomainSuspend(virDomainPtr domain)
+{
+    struct _bhyveConn *driver = domain->conn->privateData;
+    g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver);
+    virDomainObj *vm;
+    virObjectEvent *event = NULL;
+    int ret = -1;
+
+    if (!(vm = bhyveDomObjFromDomain(domain)))
+        return -1;
+
+    if (virDomainSuspendEnsureACL(domain->conn, vm->def) < 0)
+        goto cleanup;
+
+    if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), 
BHYVE_CAP_SUSPEND_RESUME)) {
+        virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+                       _("bhyve does not support suspend/resume"));
+        goto cleanup;
+    }
+
+    if (virDomainObjCheckActive(vm) < 0)
+        goto cleanup;
+
+    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) {
+        if (virBhyveProcessSuspend(driver, vm) < 0)
+            goto cleanup;
+
+        event = virDomainEventLifecycleNewFromObj(vm,
+                                                  VIR_DOMAIN_EVENT_SUSPENDED,
+                                                  
VIR_DOMAIN_EVENT_SUSPENDED_PAUSED);
+    }
+
+    if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    virObjectEventStateQueue(driver->domainEventState, event);
+    virDomainObjEndAPI(&vm);
+    return ret;
+}
+
+
+static int
+bhyveDomainResume(virDomainPtr domain)
+{
+    struct _bhyveConn *driver = domain->conn->privateData;
+    g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver);
+    virDomainObj *vm;
+    virObjectEvent *event = NULL;
+    int ret = -1;
+
+    if (!(vm = bhyveDomObjFromDomain(domain)))
+        return -1;
+
+    if (virDomainResumeEnsureACL(domain->conn, vm->def) < 0)
+        goto cleanup;
+
+    if (!virBhyveCapsGet(bhyveDriverGetBhyveCaps(driver), 
BHYVE_CAP_SUSPEND_RESUME)) {
+        virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
+                       _("bhyve does not support suspend/resume"));
+        goto cleanup;
+    }
+
+    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_PAUSED) {
+        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
+                       _("cannot resume domain not in paused state"));
+        goto cleanup;
+    }
+
+    if (virBhyveProcessResume(driver, vm) < 0)
+        goto cleanup;
+
+    event = virDomainEventLifecycleNewFromObj(vm,
+                                              VIR_DOMAIN_EVENT_RESUMED,
+                                              
VIR_DOMAIN_EVENT_RESUMED_UNPAUSED);
+
+    if (virDomainObjSave(vm, driver->xmlopt, cfg->stateDir) < 0)
+        goto cleanup;
+
+    ret = 0;
+
+ cleanup:
+    virObjectEventStateQueue(driver->domainEventState, event);
+    virDomainObjEndAPI(&vm);
+    return ret;
+}
+
+
 static int
 bhyveDomainReset(virDomainPtr domain, unsigned int flags)
 {
@@ -3334,6 +3432,8 @@ static virHypervisorDriver bhyveHypervisorDriver = {
     .domainGetGuestInfo = bhyveDomainGetGuestInfo, /* 12.7.0 */
     .domainSetLifecycleAction = bhyveDomainSetLifecycleAction, /* 12.8.0 */
     .domainReset = bhyveDomainReset, /* 12.9.0 */
+    .domainSuspend = bhyveDomainSuspend, /* 12.10.0 */
+    .domainResume = bhyveDomainResume, /* 12.10.0 */
 };
 
 
diff --git a/src/bhyve/bhyve_process.c b/src/bhyve/bhyve_process.c
index c27a5cf6ed..0cac9e841a 100644
--- a/src/bhyve/bhyve_process.c
+++ b/src/bhyve/bhyve_process.c
@@ -377,7 +377,8 @@ bhyveConnectAgent(struct _bhyveConn *driver G_GNUC_UNUSED, 
virDomainObj *vm)
 static int
 virBhyveProcessStartImpl(struct _bhyveConn *driver,
                          virDomainObj *vm,
-                         virDomainRunningReason reason)
+                         virDomainRunningReason reason,
+                         bool resume)
 {
     g_autofree char *devmap_file = NULL;
     g_autofree char *devicemap = NULL;
@@ -421,7 +422,7 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
         goto cleanup;
 
     /* Call bhyve to start the VM */
-    if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, false)))
+    if (!(cmd = virBhyveProcessBuildBhyveCmd(driver, vm->def, resume, false)))
         goto cleanup;
 
     virCommandSetOutputFD(cmd, &logfd);
@@ -493,8 +494,11 @@ virBhyveProcessStartImpl(struct _bhyveConn *driver,
         goto cleanup;
     }
 
-    /* /dev/vmm can appear before the monitor mode IPC socket is ready. */
-    if (!driver->privileged && bhyveProcessWaitForVMPid(vm) < 0)
+    /* /dev/vmm can appear before the monitor mode IPC socket is ready.
+     * A loader also creates it before bhyve starts restoring
+     * a suspended VM. */
+    if ((!driver->privileged || resume) &&
+        bhyveProcessWaitForVMPid(vm) < 0)
         goto cleanup;
 
     vm->def->id = vm->pid;
@@ -698,7 +702,7 @@ virBhyveProcessStart(bhyveConn *driver,
     if (bhyveProcessPrepareHost(driver, vm->def, flags) < 0)
         goto cleanup;
 
-    return virBhyveProcessStartImpl(driver, vm, reason);
+    return virBhyveProcessStartImpl(driver, vm, reason, false);
 
  cleanup:
     virErrorPreserveLast(&save_err);
@@ -756,9 +760,10 @@ bhyveProcessRemoveIPCSocket(struct _bhyveConn *driver,
 static int
 virBhyveProcessStopImpl(struct _bhyveConn *driver,
                         virDomainObj *vm,
-                        virDomainShutoffReason reason,
+                        int reason,
                         bool forceCleanup,
-                        bool restoreDef)
+                        bool restoreDef,
+                        bool suspend)
 {
     int ret = 0;
     size_t i = 0;
@@ -787,6 +792,20 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver,
         }
     }
 
+    if (suspend) {
+        g_autofree char *snapshot_dir = 
g_build_filename(driver->config->snapshotDir,
+                                                        vm->def->name, NULL);
+
+        if (g_mkdir_with_parents(snapshot_dir, 0700) < 0) {
+            virReportSystemError(errno,
+                                 _("Failed to mkdir %1$s"), snapshot_dir);
+            ret = -1;
+            if (!save_err)
+                virErrorPreserveLast(&save_err);
+            goto cleanup;
+        }
+    }
+
     /* Destroy monitor before running the actual destroy command to prevent
      * it from detecting VM shutdown and entering this cleanup routine again */
     if ((priv != NULL) && (priv->mon != NULL))
@@ -794,7 +813,10 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver,
 
     virBhyveDomainObjStopWorker(vm);
 
-    cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def);
+    if (suspend)
+        cmd = virBhyveProcessBuildSuspendCmd(driver, vm->def);
+    else
+        cmd = virBhyveProcessBuildDestroyCmd(driver, vm->def);
     if (virCommandRun(cmd, NULL) < 0) {
         /* Only failure of the actual destroy command warrants unsuccessful 
return code,
          * other failures are not considered critical */
@@ -832,7 +854,9 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver,
 
     virCloseCallbacksDomainRemove(vm, NULL, bhyveProcessAutoDestroy);
 
-    virDomainObjSetState(vm, VIR_DOMAIN_SHUTOFF, reason);
+    virDomainObjSetState(vm,
+                         suspend ? VIR_DOMAIN_PAUSED : VIR_DOMAIN_SHUTOFF,
+                         reason);
     vm->pid = 0;
     vm->def->id = -1;
 
@@ -842,7 +866,7 @@ virBhyveProcessStopImpl(struct _bhyveConn *driver,
     bhyveProcessRemoveDomainStatus(driver->config->stateDir, vm->def->name);
     bhyveProcessRemoveIPCSocket(driver, vm->def->name);
 
-    if (restoreDef)
+    if (restoreDef && !suspend)
         virDomainObjRemoveTransientDef(vm);
 
  cleanup:
@@ -856,7 +880,16 @@ virBhyveProcessStop(struct _bhyveConn *driver,
                     virDomainShutoffReason reason,
                     bool forceCleanup)
 {
-    return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true);
+    return virBhyveProcessStopImpl(driver, vm, reason, forceCleanup, true,
+                                   false);
+}
+
+int
+virBhyveProcessSuspend(struct _bhyveConn *driver,
+                       virDomainObj *vm)
+{
+    return virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_PAUSED_USER,
+                                   false, false, true);
 }
 
 int
@@ -885,10 +918,11 @@ virBhyveProcessRestart(struct _bhyveConn *driver,
                        virDomainObj *vm)
 {
     if (virBhyveProcessStopImpl(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN,
-                                false, false) < 0)
+                                false, false, false) < 0)
         return -1;
 
-    if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED) < 0)
+    if (virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_BOOTED,
+                                 false) < 0)
         return -1;
 
     return 0;
@@ -905,6 +939,14 @@ virBhyveProcessReset(struct _bhyveConn *driver, 
virDomainObj *vm)
     return 0;
 }
 
+int
+virBhyveProcessResume(struct _bhyveConn *driver,
+                      virDomainObj *vm)
+{
+    return virBhyveProcessStartImpl(driver, vm, VIR_DOMAIN_RUNNING_UNPAUSED,
+                                    true);
+}
+
 int
 virBhyveGetDomainTotalCpuStats(virDomainObj *vm,
                                unsigned long long *cpustats)
diff --git a/src/bhyve/bhyve_process.h b/src/bhyve/bhyve_process.h
index 2f0ce21167..655121a150 100644
--- a/src/bhyve/bhyve_process.h
+++ b/src/bhyve/bhyve_process.h
@@ -47,12 +47,18 @@ int virBhyveProcessStop(struct _bhyveConn *driver,
                         virDomainShutoffReason reason,
                         bool forceCleanup);
 
+int virBhyveProcessSuspend(struct _bhyveConn *driver,
+                           virDomainObj *vm);
+
 int virBhyveProcessRestart(struct _bhyveConn *driver,
                            virDomainObj *vm);
 
 int virBhyveProcessReset(struct _bhyveConn *driver,
                          virDomainObj *vm);
 
+int virBhyveProcessResume(struct _bhyveConn *driver,
+                          virDomainObj *vm);
+
 int virBhyveProcessShutdown(virDomainObj *vm);
 
 pid_t virBhyveProcessGetBhyvePid(virDomainObj *vm);
diff --git a/src/bhyve/bhyve_utils.h b/src/bhyve/bhyve_utils.h
index 6d15035693..fce67dfdf3 100644
--- a/src/bhyve/bhyve_utils.h
+++ b/src/bhyve/bhyve_utils.h
@@ -42,6 +42,7 @@ struct _virBhyveDriverConfig {
     char *logDir;
     char *nvramDir;
     char *stateDir;
+    char *snapshotDir;
     char *bhyveRunDir;
 
     char *ubootPath;
diff --git a/tests/bhyvexml2argvtest.c b/tests/bhyvexml2argvtest.c
index f5df8a0265..078d5faae1 100644
--- a/tests/bhyvexml2argvtest.c
+++ b/tests/bhyvexml2argvtest.c
@@ -70,7 +70,7 @@ static int testCompareXMLToArgvFiles(const char *xml,
         goto out;
     }
 
-    cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false);
+    cmd = virBhyveProcessBuildBhyveCmd(&driver, vm->def, false, false);
     if (vm->def->os.loader)
         ldcmd = virCommandNew("dummy");
     else
-- 
2.56.0

Reply via email to