>for mapping disks/NICs I see three options: >>- global switch to map everything to one target, with fall-back to keep >>as-is (current) >>- mapping of source storage/bridge to target storage/bridge >>- mapping of source disks (volid)/NIC to target storage/bridge
I like the second option "mapping of source storage/bridge to target storage/bridge" vmware is also doing it like this https://www.starwindsoftware.com/blog/vsphere-6-7-hot-migrate-vms-to-different-clusters-with-vmotion Also, it could be great to save mapping for reusing later ----- Mail original ----- De: "Fabian Grünbichler" <f.gruenbich...@proxmox.com> À: "pve-devel" <pve-devel@pve.proxmox.com> Envoyé: Vendredi 6 Mars 2020 11:20:35 Objet: [pve-devel] [PATCH qemu-server 4/4] implement PoC migration to remote cluster/node there's obviously lots of TODOs and FIXMEs in here, the big ones are: - better handling of storage switching - handling of network switching - implementing cleanup - actually dropping the local/source config and disks - NBD client side is kept open by Qemu, so we need to terminate the data tunnel in order to stop the VM I think it would make sense to re-factor vm_start and pass in all the (mostly) migration-related parameters as a hash. that way, we could tell the remote side that this is a remote incoming migration, and split out the disk/network updating into its own step in mtunnel that can happen before, and then start NBD for all disks, not just local, newly allocated ones. for mapping disks/NICs I see three options: - global switch to map everything to one target, with fall-back to keep as-is (current) - mapping of source storage/bridge to target storage/bridge - mapping of source disks (volid)/NIC to target storage/bridge the intention of sending this in the current state is mainly to get feedback early on on whether we want to go down this route, or use a totally different approach. Signed-off-by: Fabian Grünbichler <f.gruenbich...@proxmox.com> --- PVE/QemuMigrate.pm | 510 +++++++++++++++++++++++++++++++++------------ 1 file changed, 376 insertions(+), 134 deletions(-) diff --git a/PVE/QemuMigrate.pm b/PVE/QemuMigrate.pm index 6db5e62..01a5b12 100644 --- a/PVE/QemuMigrate.pm +++ b/PVE/QemuMigrate.pm @@ -3,9 +3,18 @@ package PVE::QemuMigrate; use strict; use warnings; use PVE::AbstractMigrate; + +use Data::Dumper; + use IO::File; +use IO::Socket::UNIX; use IPC::Open2; +use JSON; +use MIME::Base64; use POSIX qw( WNOHANG ); +use URI::Escape; + +use PVE::APIClient::LWP; use PVE::INotify; use PVE::Tools; use PVE::Cluster; @@ -15,9 +24,11 @@ use PVE::QemuServer::Machine; use PVE::QemuServer::Monitor qw(mon_cmd); use Time::HiRes qw( usleep ); use PVE::RPCEnvironment; +use PVE::RemoteConfig; use PVE::ReplicationConfig; use PVE::ReplicationState; use PVE::Replication; +use PVE::WebSocket; use base qw(PVE::AbstractMigrate); @@ -56,8 +67,8 @@ sub finish_command_pipe { my $writer = $cmdpipe->{writer}; my $reader = $cmdpipe->{reader}; - $writer->close(); - $reader->close(); + $writer->close() if defined($writer); + $reader->close() if defined($reader); my $collect_child_process = sub { my $res = waitpid($cpid, WNOHANG); @@ -173,6 +184,109 @@ sub fork_tunnel { return $tunnel; } +sub fork_websocket_tunnel { + my ($self) = @_; + + my $remote = $self->{opts}->{remote}; + my $remote_config = PVE::RemoteConfig->new(); + my ($ips, $conn_args) = $remote_config->get_remote_info($self->{opts}->{remote}, $self->{node}); + my $conn = PVE::APIClient::LWP->new(%$conn_args); + + my $mtunnel_worker = $conn->post("api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnel", { version => 2 }); + $self->log('info', "Spawned migration tunnel worker on '$self->{opts}->{remote}/$self->{node}'"); + # FIXME add mtunnel ticket + my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape("/run/qemu-server/$self->{vmid}.mtunnel"); + + # TODO use migration network? + my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path); + + my $auth_header = "Authorization: $conn->{apitoken}"; + + $ws->connect($auth_header); + + my $reader = IO::Pipe->new(); + my $writer = IO::Pipe->new(); + + my $cpid = fork(); + if ($cpid) { + my $tunnel = { writer => $writer->writer(), reader => $reader->reader(), socket => $ws, pid => $cpid, version => 2 }; + # FIXME: read version + print "hello: '".$self->read_tunnel($tunnel, 10)."'\n"; + print "hello: '".$self->read_tunnel($tunnel, 10)."'\n"; + $self->{api_conn} = $conn; + $self->{data_tunnels} = []; + return $tunnel; + } else { + $ws->{reader} = $writer->reader(); + $ws->{writer} = $reader->writer(); + $ws->process(); + exit 0; + } +} + +sub fork_websocket_data_tunnel { + my ($self, $local, $remote) = @_; + # TODO implement TCP? + + unlink $local; + my $local_socket = IO::Socket::UNIX->new( + Type => SOCK_STREAM(), + Local => $local, + Listen => 5, + ); + + die "could not bind to local socket '$local' of data tunnel - $!\n" + if !$local_socket; + + my $cpid = fork(); + if ($cpid) { + $local_socket->close(); + return { local => $local, pid => $cpid }; + } else { + $self->log('info', "forked websocket data tunnel $local!"); + my $clients = []; + my $exit_handler = sub { + $self->log('info', "Closing data tunnel '$local'"); + $local_socket->close(); + foreach my $c (@$clients) { + $self->log('info', "closing client socket for $local #$c->{index}"); + $self->finish_command_pipe($c); + } + $self->log('info', "Closed data tunnel '$local'"); + }; + local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = $exit_handler; + my $index = 0; + while (my $client = $local_socket->accept()) { + $index++; + $self->log('info', "accepted new connection #$index on $local, establishing web socket connection..\n"); + $cpid = fork(); + if ($cpid) { + $client->close(); + push @$clients, { pid => $cpid, index => $index }; + } else { + $local_socket->close(); + local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = 'IGNORE'; + my $conn = $self->{api_conn}; + my $api_path = "/api2/json/nodes/$self->{node}/qemu/$self->{vmid}/mtunnelwebsocket?socket=".uri_escape($remote); + my $ws = PVE::WebSocket->new($conn->{host}, 8006, $api_path); + + my $auth_header = "Authorization: $conn->{apitoken}"; + + $ws->connect($auth_header); + local $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub { $ws->close() }; + + $self->log('info', "connected websocket data tunnel $local #$index"); + $ws->{reader} = $client; + $ws->{writer} = $client; + $ws->process(); + $self->log('info', "processing finished on websocket data tunnel $local #$index.."); + exit 0; + } + } + exit 0; + } +} + sub finish_tunnel { my ($self, $tunnel) = @_; @@ -239,6 +353,12 @@ sub prepare { my $targetsid = $self->{opts}->{targetstorage} // $sid; my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid); + + if ($self->{opts}->{remote}) { + push @$need_activate, $volid; + next; + } + PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}); if ($scfg->{shared}) { @@ -256,10 +376,16 @@ sub prepare { # activate volumes PVE::Storage::activate_volumes($self->{storecfg}, $need_activate); - # test ssh connection - my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ]; - eval { $self->cmd_quiet($cmd); }; - die "Can't connect to destination address using public key\n" if $@; + if ($self->{opts}->{remote}) { + # test web socket connection + $self->{tunnel} = $self->fork_websocket_tunnel(); + print "websocket tunnel started\n"; + } else { + # test ssh connection + my $cmd = [ @{$self->{rem_ssh}}, '/bin/true' ]; + eval { $self->cmd_quiet($cmd); }; + die "Can't connect to destination address using public key\n" if $@; + } return $running; } @@ -296,7 +422,7 @@ sub sync_disks { my @sids = PVE::Storage::storage_ids($self->{storecfg}); foreach my $storeid (@sids) { my $scfg = PVE::Storage::storage_config($self->{storecfg}, $storeid); - next if $scfg->{shared}; + next if $scfg->{shared} && !$self->{opts}->{remote}; next if !PVE::Storage::storage_check_enabled($self->{storecfg}, $storeid, undef, 1); # get list from PVE::Storage (for unused volumes) @@ -307,7 +433,8 @@ sub sync_disks { my $targetsid = $override_targetsid // $storeid; # check if storage is available on target node - PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}); + PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}) + if !$self->{opts}->{remote}; PVE::Storage::foreach_volid($dl, sub { my ($volid, $sid, $volname) = @_; @@ -345,9 +472,10 @@ sub sync_disks { my $targetsid = $override_targetsid // $sid; # check if storage is available on both nodes my $scfg = PVE::Storage::storage_check_node($self->{storecfg}, $sid); - PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}); + PVE::Storage::storage_check_node($self->{storecfg}, $targetsid, $self->{node}) + if !$self->{opts}->{remote}; - return if $scfg->{shared}; + return if $scfg->{shared} && !$self->{opts}->{remote}; $local_volumes->{$volid}->{ref} = $attr->{referenced_in_config} ? 'config' : 'snapshot'; @@ -423,6 +551,9 @@ sub sync_disks { my $migratable = $scfg->{type} =~ /^(?:dir|zfspool|lvmthin|lvm)$/; + # TODO: implement properly + $migratable = 1 if $self->{opts}->{remote}; + die "can't migrate '$volid' - storage type '$scfg->{type}' not supported\n" if !$migratable; @@ -435,6 +566,9 @@ sub sync_disks { my $rep_cfg = PVE::ReplicationConfig->new(); if (my $jobcfg = $rep_cfg->find_local_replication_job($vmid, $self->{node})) { die "can't live migrate VM with replicated volumes\n" if $self->{running}; + die "can't migrate VM with replicated volumes to remote cluster/node\n" + if $self->{opts}->{remote}; + $self->log('info', "replicating disk images"); my $start_time = time(); my $logfunc = sub { $self->log('info', shift) }; @@ -469,6 +603,9 @@ sub sync_disks { next; } else { next if $self->{replicated_volumes}->{$volid}; + die "storage migration to remote cluster/node not implemented yet\n" + if $self->{opts}->{remote}; + push @{$self->{volumes}}, $volid; my $opts = $self->{opts}; my $insecure = $opts->{migration_type} eq 'insecure'; @@ -489,8 +626,12 @@ sub sync_disks { sub cleanup_remotedisks { my ($self) = @_; - foreach my $target_drive (keys %{$self->{target_drive}}) { + if ($self->{opts}->{remote}) { + warn "cleanup not yet implemented"; + return; + } + foreach my $target_drive (keys %{$self->{target_drive}}) { my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr}); my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}); @@ -520,6 +661,32 @@ sub phase1 { # sync_disks fixes disk sizes to match their actual size, write changes so # target allocates correct volumes PVE::QemuConfig->write_config($vmid, $conf); + + if ($self->{opts}->{remote}) { + # TODO without disks here, send disks + nics as separate commands? + # we want single line here! + my $remote_conf = PVE::QemuConfig->load_config($vmid); + if (my $targetsid = $self->{opts}->{targetstorage}) { + PVE::QemuServer::foreach_drive($remote_conf, sub { + my ($ds, $drive) = @_; + + return if PVE::QemuServer::drive_is_cdrom($drive); + + my $volid = $drive->{file}; + return if !$volid; + return if ! grep { $_ eq $volid} @{$self->{online_local_volumes}}; + + $self->log('info', "Rewriting config to move '$ds' - '$volid' to '$targetsid'"); + + my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid); + $drive->{file} = "$targetsid:$volname"; + $remote_conf->{$ds} = PVE::QemuServer::print_drive($drive); + }); + } + my $conf_str = MIME::Base64::encode_base64url(JSON::encode_json($remote_conf)); + $self->write_tunnel($self->{tunnel}, 10, "config $conf_str"); + } + }; sub phase1_cleanup { @@ -549,13 +716,9 @@ sub phase2 { $self->log('info', "starting VM $vmid on remote node '$self->{node}'"); - my $raddr; - my $rport; - my $ruri; # the whole migration dst. URI (protocol:address[:port]) - my $nodename = PVE::INotify::nodename(); + my $info = {}; - ## start on remote node - my $cmd = [@{$self->{rem_ssh}}]; + my $nodename = PVE::INotify::nodename(); my $spice_ticket; if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { @@ -563,113 +726,145 @@ sub phase2 { $spice_ticket = $res->{ticket}; } - push @$cmd , 'qm', 'start', $vmid, '--skiplock', '--migratedfrom', $nodename; - my $migration_type = $self->{opts}->{migration_type}; + my $state_uri = $migration_type eq 'insecure' ? 'tcp' : 'unix'; - push @$cmd, '--migration_type', $migration_type; - - push @$cmd, '--migration_network', $self->{opts}->{migration_network} - if $self->{opts}->{migration_network}; - - if ($migration_type eq 'insecure') { - push @$cmd, '--stateuri', 'tcp'; + ## start on remote node + if ($self->{opts}->{remote}) { + die "insecure migration to remote cluster/node not implemented yet\n" + if $migration_type eq 'insecure'; + + my $start_json = JSON::encode_json({ + spice => $spice_ticket, + migration_type => $migration_type, + state_uri => $state_uri, + network => $self->{opts}->{migration_network}, + machine => $self->{forcemachine}, + targetstorage => $self->{opts}->{targetstorage}, + }); + $self->write_tunnel($self->{tunnel}, 10, "start $start_json"); + $info = JSON::decode_json($self->read_tunnel($self->{tunnel}, 10)); + print Dumper($info), "\n"; } else { - push @$cmd, '--stateuri', 'unix'; - } + my $cmd = [@{$self->{rem_ssh}}]; + push @$cmd , 'qm', 'start', $vmid; + push @$cmd, '--skiplock', '--migratedfrom', $nodename; + push @$cmd, '--migration_type', $migration_type; + push @$cmd, '--migration_network', $self->{opts}->{migration_network} + if $self->{opts}->{migration_network}; - if ($self->{forcemachine}) { - push @$cmd, '--machine', $self->{forcemachine}; - } + push @$cmd, '--stateuri', $state_uri; - if ($self->{online_local_volumes}) { - push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1'); - } + if ($self->{forcemachine}) { + push @$cmd, '--machine', $self->{forcemachine}; + } - my $spice_port; + if ($self->{online_local_volumes}) { + push @$cmd, '--targetstorage', ($self->{opts}->{targetstorage} // '1'); + } - # Note: We try to keep $spice_ticket secret (do not pass via command line parameter) - # instead we pipe it through STDIN - my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub { - my $line = shift; + # Note: We try to keep $spice_ticket secret (do not pass via command line parameter) + # instead we pipe it through STDIN + my $exitcode = PVE::Tools::run_command($cmd, input => $spice_ticket, outfunc => sub { + my $line = shift; - if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { - $raddr = $1; - $rport = int($2); - $ruri = "tcp:$raddr:$rport"; - } - elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) { - $raddr = $1; - die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2; - $ruri = "unix:$raddr"; - } - elsif ($line =~ m/^migration listens on port (\d+)$/) { - $raddr = "localhost"; - $rport = int($1); - $ruri = "tcp:$raddr:$rport"; - } - elsif ($line =~ m/^spice listens on port (\d+)$/) { - $spice_port = int($1); - } - elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) { - my $drivestr = $4; - my $nbd_uri = "nbd:$1:$2:exportname=$3"; - my $targetdrive = $3; - $targetdrive =~ s/drive-//g; + if ($line =~ m/^migration listens on tcp:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+)$/) { + $info->{raddr} = $1; + $info->{rport} = int($2); + $info->{ruri} = "tcp:$info->{raddr}:$info->{rport}"; + } + elsif ($line =~ m!^migration listens on unix:(/run/qemu-server/(\d+)\.migrate)$!) { + $info->{raddr} = $1; + die "Destination UNIX sockets VMID does not match source VMID" if $vmid ne $2; + $info->{ruri} = "unix:$info->{raddr}"; + } + elsif ($line =~ m/^migration listens on port (\d+)$/) { + $info->{raddr} = "localhost"; + $info->{rport} = int($1); + $info->{ruri} = "tcp:$info->{raddr}:$info->{rport}"; + } + elsif ($line =~ m/^spice listens on port (\d+)$/) { + $info->{spice_port} = int($1); + } + elsif ($line =~ m/^storage migration listens on nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+) volume:(\S+)$/) { + my $drivestr = $4; + my $nbd_uri = "nbd:$1:$2:exportname=$3"; + my $targetdrive = $3; + $targetdrive =~ s/drive-//g; - $self->{target_drive}->{$targetdrive}->{drivestr} = $drivestr; - $self->{target_drive}->{$targetdrive}->{nbd_uri} = $nbd_uri; + $info->{drives}->{$targetdrive}->{drivestr} = $drivestr; + $info->{drives}->{$targetdrive}->{nbd_uri} = $nbd_uri; - } elsif ($line =~ m/^QEMU: (.*)$/) { - $self->log('info', "[$self->{node}] $1\n"); - } - }, errfunc => sub { - my $line = shift; - $self->log('info', "[$self->{node}] $line"); - }, noerr => 1); + } elsif ($line =~ m/^QEMU: (.*)$/) { + $self->log('info', "[$self->{node}] $1\n"); + } + }, errfunc => sub { + my $line = shift; + $self->log('info', "[$self->{node}] $line"); + }, noerr => 1); - die "remote command failed with exit code $exitcode\n" if $exitcode; + die "remote command failed with exit code $exitcode\n" if $exitcode; + } - die "unable to detect remote migration address\n" if !$raddr; + die "unable to detect remote migration address\n" if !$info->{raddr}; - $self->log('info', "start remote tunnel"); + foreach my $drive (keys %{$info->{drives}}) { + $self->{target_drive}->{$drive}->{drivestr} = $info->{drives}->{$drive}->{drivestr}; + $self->{target_drive}->{$drive}->{nbd_uri} = $info->{drives}->{$drive}->{nbd_uri}; + } - if ($migration_type eq 'secure') { + if ($self->{tunnel} && $self->{opts}->{remote}) { + # TODO support TCP? + if ($info->{ruri} =~ /^unix:/) { + $self->log('info', "starting memory migration tunnel '$info->{raddr}' => '$info->{raddr}'"); + push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{raddr}, $info->{raddr}); + } else { + die "unsupported migration uri '$info->{ruri}'\n"; + } + if ($info->{nbd}) { + $self->log('info', "starting local disk migration tunnel '$info->{nbd}' => '$info->{nbd}'"); + # keep open - we might have multiple disks! + push @{$self->{data_tunnels}}, $self->fork_websocket_data_tunnel($info->{nbd}, $info->{nbd}, 1); + } + } else { + $self->log('info', "start remote tunnel"); + + if ($migration_type eq 'secure') { + if ($info->{ruri} =~ /^unix:/) { + unlink $info->{raddr}; + $self->{tunnel} = $self->fork_tunnel("$info->{raddr}:$info->{raddr}"); + $self->{tunnel}->{sock_addr} = $info->{raddr}; + + my $unix_socket_try = 0; # wait for the socket to become ready + while (! -S $info->{raddr}) { + $unix_socket_try++; + if ($unix_socket_try > 100) { + $self->{errors} = 1; + $self->finish_tunnel($self->{tunnel}); + die "Timeout, migration socket $info->{ruri} did not get ready"; + } - if ($ruri =~ /^unix:/) { - unlink $raddr; - $self->{tunnel} = $self->fork_tunnel("$raddr:$raddr"); - $self->{tunnel}->{sock_addr} = $raddr; + usleep(50000); + } - my $unix_socket_try = 0; # wait for the socket to become ready - while (! -S $raddr) { - $unix_socket_try++; - if ($unix_socket_try > 100) { - $self->{errors} = 1; - $self->finish_tunnel($self->{tunnel}); - die "Timeout, migration socket $ruri did not get ready"; + } elsif ($info->{ruri} =~ /^tcp:/) { + my $tunnel_addr; + if ($info->{raddr} eq "localhost") { + # for backwards compatibility with older qemu-server versions + my $pfamily = PVE::Tools::get_host_address_family($nodename); + my $lport = PVE::Tools::next_migrate_port($pfamily); + $tunnel_addr = "$lport:localhost:$info->{rport}"; } - usleep(50000); - } + $self->{tunnel} = $self->fork_tunnel($tunnel_addr); - } elsif ($ruri =~ /^tcp:/) { - my $tunnel_addr; - if ($raddr eq "localhost") { - # for backwards compatibility with older qemu-server versions - my $pfamily = PVE::Tools::get_host_address_family($nodename); - my $lport = PVE::Tools::next_migrate_port($pfamily); - $tunnel_addr = "$lport:localhost:$rport"; + } else { + die "unsupported protocol in migration URI: $info->{ruri}\n"; } - - $self->{tunnel} = $self->fork_tunnel($tunnel_addr); - } else { - die "unsupported protocol in migration URI: $ruri\n"; + #fork tunnel for insecure migration, to send faster commands like resume + $self->{tunnel} = $self->fork_tunnel(); } - } else { - #fork tunnel for insecure migration, to send faster commands like resume - $self->{tunnel} = $self->fork_tunnel(); } my $start = time(); @@ -688,10 +883,13 @@ sub phase2 { my $nbd_uri = $target->{nbd_uri}; my $source_drive = PVE::QemuServer::parse_drive($drive, $conf->{$drive}); - my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr}); - my $source_sid = PVE::Storage::Plugin::parse_volume_id($source_drive->{file}); - my $target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file}); + + my $target_sid; + if (!$self->{opts}->{remote}) { + my $target_drive = PVE::QemuServer::parse_drive($drive, $target->{drivestr}); + $target_sid = PVE::Storage::Plugin::parse_volume_id($target_drive->{file}); + } my $bwlimit = PVE::Storage::get_bandwidth_limit('migrate', [$source_sid, $target_sid], $opt_bwlimit); @@ -700,7 +898,7 @@ sub phase2 { } } - $self->log('info', "starting online/live migration on $ruri"); + $self->log('info', "starting online/live migration on $info->{ruri}"); $self->{livemigration} = 1; # load_defaults @@ -755,7 +953,7 @@ sub phase2 { }; $self->log('info', "migrate-set-parameters error: $@") if $@; - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga})) { + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && !$self->{opts}->{remote}) { my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); @@ -768,19 +966,19 @@ sub phase2 { eval { mon_cmd($vmid, "client_migrate_info", protocol => 'spice', - hostname => $proxyticket, 'port' => 0, 'tls-port' => $spice_port, + hostname => $proxyticket, 'port' => 0, 'tls-port' => $info->{spice_port}, 'cert-subject' => $subject); }; $self->log('info', "client_migrate_info error: $@") if $@; } - $self->log('info', "start migrate command to $ruri"); + $self->log('info', "start migrate command to $info->{ruri}"); eval { - mon_cmd($vmid, "migrate", uri => $ruri); + mon_cmd($vmid, "migrate", uri => $info->{ruri}); }; my $merr = $@; - $self->log('info', "migrate uri => $ruri failed: $merr") if $merr; + $self->log('info', "migrate uri => $info->{ruri} failed: $merr") if $merr; my $lstat = 0; my $usleep = 1000000; @@ -918,11 +1116,15 @@ sub phase2_cleanup { my $nodename = PVE::INotify::nodename(); - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + if ($self->{tunnel} && $self->{tunnel}->{version} >= 2) { + $self->write_tunnel($self->{tunnel}, 10, "stop"); + } else { + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'stop', $vmid, '--skiplock', '--migratedfrom', $nodename]; + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } } if ($self->{tunnel}) { @@ -941,6 +1143,10 @@ sub phase3 { return if $self->{phase2errors}; # destroy local copies + + # FIXME remove early return for real migration! + return; + foreach my $volid (@$volids) { eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; if (my $err = $@) { @@ -968,35 +1174,46 @@ sub phase3_cleanup { eval { PVE::QemuMigrate::cleanup_remotedisks($self) }; die "Failed to complete storage migration: $err\n"; } else { + # TODO handle properly + if (!$self->{opts}->{remote}) { foreach my $target_drive (keys %{$self->{target_drive}}) { my $drive = PVE::QemuServer::parse_drive($target_drive, $self->{target_drive}->{$target_drive}->{drivestr}); $conf->{$target_drive} = PVE::QemuServer::print_drive($drive); PVE::QemuConfig->write_config($vmid, $conf); } + } } } # transfer replication state before move config $self->transfer_replication_state() if $self->{replicated_volumes}; - # move config to remote node - my $conffile = PVE::QemuConfig->config_file($vmid); - my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node}); + if ($self->{opts}->{remote}) { + # TODO delete local config? + } else { + # move config to remote node + my $conffile = PVE::QemuConfig->config_file($vmid); + my $newconffile = PVE::QemuConfig->config_file($vmid, $self->{node}); - die "Failed to move config to node '$self->{node}' - rename failed: $!\n" - if !rename($conffile, $newconffile); + die "Failed to move config to node '$self->{node}' - rename failed: $!\n" + if !rename($conffile, $newconffile); + } $self->switch_replication_job_target() if $self->{replicated_volumes}; if ($self->{livemigration}) { if ($self->{storage_migration}) { - # stop nbd server on remote vm - requirement for resume since 2.9 - my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid]; + if ($tunnel && $tunnel->{version} && $tunnel->{version} >= 2) { + $self->write_tunnel($tunnel, 30, 'nbdstop'); + } else { + # stop nbd server on remote vm - requirement for resume since 2.9 + my $cmd = [@{$self->{rem_ssh}}, 'qm', 'nbdstop', $vmid]; - eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + eval{ PVE::Tools::run_command($cmd, outfunc => sub {}, errfunc => sub {}) }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } } } @@ -1030,16 +1247,30 @@ sub phase3_cleanup { # close tunnel on successful migration, on error phase2_cleanup closed it if ($tunnel) { - eval { finish_tunnel($self, $tunnel); }; - if (my $err = $@) { - $self->log('err', $err); - $self->{errors} = 1; + if ($tunnel->{version} == 1) { + eval { finish_tunnel($self, $tunnel); }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } + $tunnel = undef; + delete $self->{tunnel}; + } else { + foreach my $data_tunnel (@{$self->{data_tunnels}}) { + eval { + kill(15, $data_tunnel->{pid}); + }; + if (my $err = $@) { + $self->log('err', $err); + $self->{errors} = 1; + } + } } } eval { my $timer = 0; - if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running}) { + if (PVE::QemuServer::vga_conf_has_spice($conf->{vga}) && $self->{running} && !$self->{opts}->{remote}) { $self->log('info', "Waiting for spice server migration"); while (1) { my $res = mon_cmd($vmid, 'query-spice'); @@ -1072,6 +1303,9 @@ sub phase3_cleanup { # destroy local copies my $volids = $self->{online_local_volumes}; + # TODO remove for proper migration! + if (!$self->{opts}->{remote}) { + foreach my $volid (@$volids) { eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); }; if (my $err = $@) { @@ -1080,12 +1314,20 @@ sub phase3_cleanup { last if $err =~ /^interrupted by signal$/; } } + } } + # TODO remove local config for proper remote migration + # clear migrate lock - my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ]; - $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); + if ($tunnel) { + $self->write_tunnel($tunnel, 10, "unlock"); + $self->finish_tunnel($tunnel); + } else { + my $cmd = [ @{$self->{rem_ssh}}, 'qm', 'unlock', $vmid ]; + $self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock"); + } } sub final_cleanup { -- 2.20.1 _______________________________________________ pve-devel mailing list pve-devel@pve.proxmox.com https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel _______________________________________________ pve-devel mailing list pve-devel@pve.proxmox.com https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
