On 02/14/2017 03:34 AM, Jim Fehlig wrote: > On 02/07/2017 05:35 PM, Joao Martins wrote: >> From: Bob Liu <bob....@oracle.com> >> >> Tunnelled migration doesn't require any extra network connections beside >> the libvirt daemon. It's capable of strong encryption and the default >> option of openstack-nova. >> >> This patch adds the tunnelled migration(Tunnel3params) support to libxl. >> On the source side, the data flow is: >> >> * libxlDoMigrateSend() -> pipe libxlTunnel3MigrationFunc() polls pipe >> * out and then write to dest stream. >> >> While on the destination side: >> * Stream -> pipe -> 'recvfd of libxlDomainStartRestore' >> >> The usage is the same as p2p migration, execpt adding one extra >> '--tunnelled' to the libvirt p2p migration command. >> >> Signed-off-by: Bob Liu <bob....@oracle.com> >> Signed-off-by: Joao Martins <joao.m.mart...@oracle.com> >> --- >> v2 (comments from Jim): >> * Adjust common preparetunnel function reflecting v1 suggestions >> - Using common top-level Prepare function by adding is_tunnel variable >> - Using libxl_migration PrepareAny added in patch 1 >> - virHook support in PrepareTunnel3 >> * Move virThreadPtr from libxlTunnelMigrationThread into libxlTunnelControl >> * Rename function local variable from tmThreadPtr to arg >> * Remove redundant assignment "tmThreadPtr = &tc->tmThread;" always being >> not >> null. >> --- >> src/libxl/libxl_driver.c | 49 ++++++-- >> src/libxl/libxl_migration.c | 280 >> +++++++++++++++++++++++++++++++++++++++++--- >> src/libxl/libxl_migration.h | 9 ++ >> 3 files changed, 313 insertions(+), 25 deletions(-) >> >> diff --git a/src/libxl/libxl_driver.c b/src/libxl/libxl_driver.c >> index 7bc8adf..b34f120 100644 >> --- a/src/libxl/libxl_driver.c >> +++ b/src/libxl/libxl_driver.c >> @@ -5938,14 +5938,14 @@ libxlDomainMigratePrepareCommon(virConnectPtr dconn, >> char **cookieout ATTRIBUTE_UNUSED, >> int *cookieoutlen ATTRIBUTE_UNUSED, >> unsigned int flags, >> - void *data) >> + void *data, >> + bool is_tunnel) >> { >> libxlDriverPrivatePtr driver = dconn->privateData; >> virDomainDefPtr def = NULL; >> const char *dom_xml = NULL; >> const char *dname = NULL; >> const char *uri_in = NULL; >> - char **uri_out = data; >> >> #ifdef LIBXL_HAVE_NO_SUSPEND_RESUME >> virReportUnsupportedError(); >> @@ -5971,12 +5971,25 @@ libxlDomainMigratePrepareCommon(virConnectPtr dconn, >> if (!(def = libxlDomainMigrationPrepareDef(driver, dom_xml, dname))) >> goto error; >> >> - if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0) >> - goto error; >> + if (is_tunnel) { >> + virStreamPtr st = data; >> >> - if (libxlDomainMigrationPrepare(dconn, &def, uri_in, uri_out, >> - cookiein, cookieinlen, flags) < 0) >> - goto error; >> + if (virDomainMigratePrepareTunnel3ParamsEnsureACL(dconn, def) < 0) >> + goto error; >> + >> + if (libxlDomainMigrationPrepareTunnel3(dconn, st, &def, cookiein, >> + cookieinlen, flags) < 0) >> + goto error; >> + } else { >> + char **uri_out = data; >> + >> + if (virDomainMigratePrepare3ParamsEnsureACL(dconn, def) < 0) >> + goto error; >> + >> + if (libxlDomainMigrationPrepare(dconn, &def, uri_in, uri_out, >> + cookiein, cookieinlen, flags) < 0) >> + goto error; >> + } > > Same comment here about the ACL checks and 'make check. With both patches > applied > > ./libxl/libxl_driver.c:5981 Mismatch check > 'virDomainMigratePrepareTunnel3ParamsEnsureACL' for function > 'libxlDomainMigratePrepareCommon' > ./libxl/libxl_driver.c:5990 Mismatch check > 'virDomainMigratePrepare3ParamsEnsureACL' for function > 'libxlDomainMigratePrepareCommon' Yeap, see my comment in the previous patch.
>> return 0; >> >> @@ -5999,7 +6012,24 @@ libxlDomainMigratePrepare3Params(virConnectPtr dconn, >> return libxlDomainMigratePrepareCommon(dconn, params, nparams, >> cookiein, cookieinlen, >> cookieout, cookieoutlen, >> - flags, uri_out); >> + flags, uri_out, false); >> +} >> + >> +static int >> +libxlDomainMigratePrepareTunnel3Params(virConnectPtr dconn, >> + virStreamPtr st, >> + virTypedParameterPtr params, >> + int nparams, >> + const char *cookiein, >> + int cookieinlen, >> + char **cookieout ATTRIBUTE_UNUSED, >> + int *cookieoutlen ATTRIBUTE_UNUSED, >> + unsigned int flags) >> +{ >> + return libxlDomainMigratePrepareCommon(dconn, params, nparams, >> + cookiein, cookieinlen, >> + cookieout, cookieoutlen, >> + flags, st, true); >> } >> >> static int >> @@ -6047,7 +6077,7 @@ libxlDomainMigratePerform3Params(virDomainPtr dom, >> if (virDomainMigratePerform3ParamsEnsureACL(dom->conn, vm->def) < 0) >> goto cleanup; >> >> - if (flags & VIR_MIGRATE_PEER2PEER) { >> + if ((flags & (VIR_MIGRATE_TUNNELLED | VIR_MIGRATE_PEER2PEER))) { >> if (libxlDomainMigrationPerformP2P(driver, vm, dom->conn, dom_xml, >> dconnuri, uri, dname, flags) < 0) >> goto cleanup; >> @@ -6532,6 +6562,7 @@ static virHypervisorDriver libxlHypervisorDriver = { >> .nodeDeviceReset = libxlNodeDeviceReset, /* 1.2.3 */ >> .domainMigrateBegin3Params = libxlDomainMigrateBegin3Params, /* 1.2.6 */ >> .domainMigratePrepare3Params = libxlDomainMigratePrepare3Params, /* >> 1.2.6 */ >> + .domainMigratePrepareTunnel3Params = >> libxlDomainMigratePrepareTunnel3Params, /* 3.1.0 */ >> .domainMigratePerform3Params = libxlDomainMigratePerform3Params, /* >> 1.2.6 */ >> .domainMigrateFinish3Params = libxlDomainMigrateFinish3Params, /* 1.2.6 >> */ >> .domainMigrateConfirm3Params = libxlDomainMigrateConfirm3Params, /* >> 1.2.6 */ >> diff --git a/src/libxl/libxl_migration.c b/src/libxl/libxl_migration.c >> index 7beabd2..3aa0797 100644 >> --- a/src/libxl/libxl_migration.c >> +++ b/src/libxl/libxl_migration.c >> @@ -44,6 +44,7 @@ >> #include "libxl_migration.h" >> #include "locking/domain_lock.h" >> #include "virtypedparam.h" >> +#include "fdstream.h" >> >> #define VIR_FROM_THIS VIR_FROM_LIBXL >> >> @@ -554,6 +555,95 @@ libxlDomainMigrationPrepareAny(virConnectPtr dconn, >> } >> >> int >> +libxlDomainMigrationPrepareTunnel3(virConnectPtr dconn, >> + virStreamPtr st, >> + virDomainDefPtr *def, >> + const char *cookiein, >> + int cookieinlen, >> + unsigned int flags) >> +{ >> + libxlMigrationCookiePtr mig = NULL; >> + libxlDriverPrivatePtr driver = dconn->privateData; >> + virDomainObjPtr vm = NULL; >> + libxlMigrationDstArgs *args = NULL; >> + virThread thread; >> + bool taint_hook = false; >> + libxlDomainObjPrivatePtr priv = NULL; >> + char *xmlout = NULL; >> + int dataFD[2] = { -1, -1 }; >> + int ret = -1; >> + >> + if (libxlDomainMigrationPrepareAny(dconn, def, cookiein, cookieinlen, >> + &mig, &xmlout, &taint_hook) < 0) >> + goto error; >> + >> + if (!(vm = virDomainObjListAdd(driver->domains, *def, >> + driver->xmlopt, >> + VIR_DOMAIN_OBJ_LIST_ADD_LIVE | >> + VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, >> + NULL))) >> + goto error; >> + >> + *def = NULL; >> + priv = vm->privateData; >> + >> + if (taint_hook) { >> + /* Domain XML has been altered by a hook script. */ >> + priv->hookRun = true; >> + } >> + >> + /* >> + * The data flow of tunnel3 migration in the dest side: >> + * stream -> pipe -> recvfd of libxlDomainStartRestore >> + */ >> + if (pipe(dataFD) < 0) >> + goto error; >> + >> + /* Stream data will be written to pipeIn */ >> + if (virFDStreamOpen(st, dataFD[1]) < 0) >> + goto error; >> + dataFD[1] = -1; /* 'st' owns the FD now & will close it */ >> + >> + if (libxlMigrationDstArgsInitialize() < 0) >> + goto error; >> + >> + if (!(args = virObjectNew(libxlMigrationDstArgsClass))) >> + goto error; >> + >> + args->conn = dconn; >> + args->vm = vm; >> + args->flags = flags; >> + args->migcookie = mig; >> + /* Receive from pipeOut */ >> + args->recvfd = dataFD[0]; >> + args->nsocks = 0; >> + if (virThreadCreate(&thread, false, libxlDoMigrateReceive, args) < 0) { >> + virReportError(VIR_ERR_OPERATION_FAILED, "%s", >> + _("Failed to create thread for receiving migration >> data")); >> + goto error; >> + } >> + >> + ret = 0; >> + goto done; >> + >> + error: >> + VIR_FORCE_CLOSE(dataFD[1]); >> + VIR_FORCE_CLOSE(dataFD[0]); >> + virObjectUnref(args); >> + /* Remove virDomainObj from domain list */ >> + if (vm) { >> + virDomainObjListRemove(driver->domains, vm); >> + vm = NULL; >> + } >> + >> + done: >> + if (vm) >> + virObjectUnlock(vm); >> + >> + return ret; >> +} >> + >> +int >> libxlDomainMigrationPrepare(virConnectPtr dconn, >> virDomainDefPtr *def, >> const char *uri_in, >> @@ -734,9 +824,148 @@ libxlDomainMigrationPrepare(virConnectPtr dconn, >> return ret; >> } >> >> -/* This function is a simplification of virDomainMigrateVersion3Full >> - * excluding tunnel support and restricting it to migration v3 >> - * with params since it was the first to be introduced in libxl. >> +typedef struct _libxlTunnelMigrationThread libxlTunnelMigrationThread; >> +struct _libxlTunnelMigrationThread { >> + virStreamPtr st; >> + int srcFD; >> +}; >> +#define TUNNEL_SEND_BUF_SIZE 65536 >> + >> +/* >> + * The data flow of tunnel3 migration in the src side: >> + * libxlDoMigrateSend() -> pipe >> + * libxlTunnel3MigrationFunc() polls pipe out and then write to dest stream. >> + */ >> +static void libxlTunnel3MigrationFunc(void *arg) >> +{ >> + libxlTunnelMigrationThread *data = (libxlTunnelMigrationThread *)arg; >> + char *buffer = NULL; >> + struct pollfd fds[1]; >> + int timeout = -1; >> + >> + if (VIR_ALLOC_N(buffer, TUNNEL_SEND_BUF_SIZE) < 0) { >> + virReportError(errno, "%s", _("poll failed in migration tunnel")); > > Error reporting is handled by VIR_ALLOC_N. True, remnant from previous patch. I've remove it. >> + return; >> + } >> + >> + fds[0].fd = data->srcFD; >> + for (;;) { >> + int ret; >> + >> + fds[0].events = POLLIN; >> + fds[0].revents = 0; >> + ret = poll(fds, ARRAY_CARDINALITY(fds), timeout); >> + if (ret < 0) { >> + if (errno == EAGAIN || errno == EINTR) >> + continue; >> + virReportError(errno, "%s", >> + _("poll failed in libxlTunnel3MigrationFunc")); >> + goto cleanup; >> + } >> + >> + if (ret == 0) { >> + VIR_DEBUG("poll got timeout"); >> + break; >> + } >> + >> + if (fds[0].revents & (POLLIN | POLLERR | POLLHUP)) { >> + int nbytes; >> + >> + nbytes = read(data->srcFD, buffer, TUNNEL_SEND_BUF_SIZE); >> + if (nbytes > 0) { >> + /* Write to dest stream */ >> + if (virStreamSend(data->st, buffer, nbytes) < 0) { >> + virReportError(errno, "%s", >> + _("tunnelled migration failed to send to >> virStream")); > > virStreamSend also reports (a more helpful?) error. Indeed :) I've removed this too. >> + virStreamAbort(data->st); >> + goto cleanup; >> + } >> + } else if (nbytes < 0) { >> + virReportError(errno, "%s", >> + _("tunnelled migration failed to read from >> xen side")); >> + virStreamAbort(data->st); >> + goto cleanup; >> + } else { >> + /* EOF; transferred all data */ >> + break; >> + } >> + } >> + } >> + >> + if (virStreamFinish(data->st) < 0) >> + virReportError(errno, "%s", >> + _("tunnelled migration failed to finish stream")); > > AFAICT virStreamFinish also reports error. OK, removed. > Besides the nits, looks good. I'll find some time tomorrow to test the > patches. > Thanks! Cool, Thanks! I'll wait for the comment on patch 2 before respinning. > >> + cleanup: >> + VIR_FREE(buffer); >> + >> + return; >> +} >> + >> +struct libxlTunnelControl { >> + libxlTunnelMigrationThread tmThread; >> + virThread thread; >> + int dataFD[2]; >> +}; >> + >> +static int >> +libxlMigrationStartTunnel(libxlDriverPrivatePtr driver, >> + virDomainObjPtr vm, >> + unsigned long flags, >> + virStreamPtr st, >> + struct libxlTunnelControl *tc) >> +{ >> + libxlTunnelMigrationThread *arg = NULL; >> + int ret = -1; >> + >> + if (VIR_ALLOC(tc) < 0) >> + goto out; >> + >> + tc->dataFD[0] = -1; >> + tc->dataFD[1] = -1; >> + if (pipe(tc->dataFD) < 0) { >> + virReportError(errno, "%s", _("Unable to make pipes")); >> + goto out; >> + } >> + >> + arg = &tc->tmThread; >> + /* Read from pipe */ >> + arg->srcFD = tc->dataFD[0]; >> + /* Write to dest stream */ >> + arg->st = st; >> + if (virThreadCreate(&tc->thread, true, >> + libxlTunnel3MigrationFunc, arg) < 0) { >> + virReportError(errno, "%s", >> + _("Unable to create tunnel migration thread")); >> + goto out; >> + } >> + >> + virObjectUnlock(vm); >> + /* Send data to pipe */ >> + ret = libxlDoMigrateSend(driver, vm, flags, tc->dataFD[1]); >> + virObjectLock(vm); >> + >> + out: >> + /* libxlMigrationStopTunnel will be called in libxlDoMigrateP2P to free >> + * all resources for us. */ >> + return ret; >> +} >> + >> +static void libxlMigrationStopTunnel(struct libxlTunnelControl *tc) >> +{ >> + if (!tc) >> + return; >> + >> + virThreadCancel(&tc->thread); >> + virThreadJoin(&tc->thread); >> + >> + VIR_FORCE_CLOSE(tc->dataFD[0]); >> + VIR_FORCE_CLOSE(tc->dataFD[1]); >> + VIR_FREE(tc); >> +} >> + >> +/* This function is a simplification of virDomainMigrateVersion3Full and >> + * restricting it to migration v3 with params since it was the first to be >> + * introduced in libxl. >> */ >> static int >> libxlDoMigrateP2P(libxlDriverPrivatePtr driver, >> @@ -761,6 +990,9 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver, >> bool cancelled = true; >> virErrorPtr orig_err = NULL; >> int ret = -1; >> + /* For tunnel migration */ >> + virStreamPtr st = NULL; >> + struct libxlTunnelControl *tc = NULL; >> >> dom_xml = libxlDomainMigrationBegin(sconn, vm, xmlin, >> &cookieout, &cookieoutlen); >> @@ -788,29 +1020,40 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver, >> >> VIR_DEBUG("Prepare3"); >> virObjectUnlock(vm); >> - ret = dconn->driver->domainMigratePrepare3Params >> - (dconn, params, nparams, cookieout, cookieoutlen, NULL, NULL, >> &uri_out, destflags); >> + if (flags & VIR_MIGRATE_TUNNELLED) { >> + if (!(st = virStreamNew(dconn, 0))) >> + goto cleanup; >> + ret = dconn->driver->domainMigratePrepareTunnel3Params >> + (dconn, st, params, nparams, cookieout, cookieoutlen, NULL, >> NULL, destflags); >> + } else { >> + ret = dconn->driver->domainMigratePrepare3Params >> + (dconn, params, nparams, cookieout, cookieoutlen, NULL, NULL, >> &uri_out, destflags); >> + } >> virObjectLock(vm); >> >> if (ret == -1) >> goto cleanup; >> >> - if (uri_out) { >> - if (virTypedParamsReplaceString(¶ms, &nparams, >> - VIR_MIGRATE_PARAM_URI, uri_out) < >> 0) { >> - orig_err = virSaveLastError(); >> + if (!(flags & VIR_MIGRATE_TUNNELLED)) { >> + if (uri_out) { >> + if (virTypedParamsReplaceString(¶ms, &nparams, >> + VIR_MIGRATE_PARAM_URI, uri_out) >> < 0) { >> + orig_err = virSaveLastError(); >> + goto finish; >> + } >> + } else { >> + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", >> + _("domainMigratePrepare3 did not set uri")); >> goto finish; >> } >> - } else { >> - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", >> - _("domainMigratePrepare3 did not set uri")); >> - goto finish; >> } >> >> VIR_DEBUG("Perform3 uri=%s", NULLSTR(uri_out)); >> - ret = libxlDomainMigrationPerform(driver, vm, NULL, NULL, >> - uri_out, NULL, flags); >> - >> + if (flags & VIR_MIGRATE_TUNNELLED) >> + ret = libxlMigrationStartTunnel(driver, vm, flags, st, tc); >> + else >> + ret = libxlDomainMigrationPerform(driver, vm, NULL, NULL, >> + uri_out, NULL, flags); >> if (ret < 0) >> orig_err = virSaveLastError(); >> >> @@ -848,6 +1091,11 @@ libxlDoMigrateP2P(libxlDriverPrivatePtr driver, >> vm->def->name); >> >> cleanup: >> + if (flags & VIR_MIGRATE_TUNNELLED) { >> + libxlMigrationStopTunnel(tc); >> + virObjectUnref(st); >> + } >> + >> if (ddomain) { >> virObjectUnref(ddomain); >> ret = 0; >> diff --git a/src/libxl/libxl_migration.h b/src/libxl/libxl_migration.h >> index 8a074a0..fcea558 100644 >> --- a/src/libxl/libxl_migration.h >> +++ b/src/libxl/libxl_migration.h >> @@ -29,6 +29,7 @@ >> # define LIBXL_MIGRATION_FLAGS \ >> (VIR_MIGRATE_LIVE | \ >> VIR_MIGRATE_PEER2PEER | \ >> + VIR_MIGRATE_TUNNELLED | \ >> VIR_MIGRATE_PERSIST_DEST | \ >> VIR_MIGRATE_UNDEFINE_SOURCE | \ >> VIR_MIGRATE_PAUSED) >> @@ -53,6 +54,14 @@ libxlDomainMigrationPrepareDef(libxlDriverPrivatePtr >> driver, >> const char *dname); >> >> int >> +libxlDomainMigrationPrepareTunnel3(virConnectPtr dconn, >> + virStreamPtr st, >> + virDomainDefPtr *def, >> + const char *cookiein, >> + int cookieinlen, >> + unsigned int flags); >> + >> +int >> libxlDomainMigrationPrepare(virConnectPtr dconn, >> virDomainDefPtr *def, >> const char *uri_in, >> > -- libvir-list mailing list libvir-list@redhat.com https://www.redhat.com/mailman/listinfo/libvir-list