Factor out common code, which will be used by the new API endpoint to join a cluster and the old legacy SSH method which we will keep for a bit.
This is not a completely 1:1 move: * change &$error(...) to $error->(...) * removing a few empty lines, where code was so spread out that those lines resulted in the opposite of what they intended, i.e., less readability * rename $backupdir to $dbbackupdir Signed-off-by: Thomas Lamprecht <[email protected]> --- data/PVE/CLI/pvecm.pm | 154 ++------------------------------------------------ data/PVE/Cluster.pm | 146 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 149 deletions(-) diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm index a92f831..ca7061f 100755 --- a/data/PVE/CLI/pvecm.pm +++ b/data/PVE/CLI/pvecm.pm @@ -21,44 +21,8 @@ $ENV{HOME} = '/root'; # for ssh-copy-id my $basedir = "/etc/pve"; my $clusterconf = "$basedir/corosync.conf"; my $libdir = "/var/lib/pve-cluster"; -my $backupdir = "/var/lib/pve-cluster/backup"; -my $dbfile = "$libdir/config.db"; my $authfile = "/etc/corosync/authkey"; -sub backup_database { - - print "backup old database\n"; - - mkdir $backupdir; - - my $ctime = time(); - my $cmd = [ - ['echo', '.dump'], - ['sqlite3', $dbfile], - ['gzip', '-', \ ">${backupdir}/config-${ctime}.sql.gz"], - ]; - - run_command($cmd, 'errmsg' => "cannot backup old database\n"); - - # purge older backup - my $maxfiles = 10; - - my @bklist = (); - foreach my $fn (<$backupdir/config-*.sql.gz>) { - if ($fn =~ m!/config-(\d+)\.sql.gz$!) { - push @bklist, [$fn, $1]; - } - } - - @bklist = sort { $b->[1] <=> $a->[1] } @bklist; - - while (scalar (@bklist) >= $maxfiles) { - my $d = pop @bklist; - print "delete old backup '$d->[0]'\n"; - unlink $d->[0]; - } -} - __PACKAGE__->register_method ({ name => 'keygen', @@ -300,67 +264,10 @@ __PACKAGE__->register_method ({ PVE::Cluster::setup_rootsshconfig(); PVE::Cluster::setup_ssh_keys(); + PVE::Cluster::assert_joinable($param->{ring0_addr}, $param->{ring1_addr}, $param->{force}); + my $host = $param->{hostname}; - my ($errors, $warnings) = ('', ''); - - my $error = sub { - my ($msg, $suppress) = @_; - - if ($suppress) { - $warnings .= "* $msg\n"; - } else { - $errors .= "* $msg\n"; - } - }; - - if (!$param->{force}) { - - if (-f $authfile) { - &$error("authentication key '$authfile' already exists", $param->{force}); - } - - if (-f $clusterconf) { - &$error("cluster config '$clusterconf' already exists", $param->{force}); - } - - my $vmlist = PVE::Cluster::get_vmlist(); - if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) { - &$error("this host already contains virtual guests", $param->{force}); - } - - if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) { - &$error("corosync is already running, is this node already in a cluster?!", $param->{force}); - } - } - - # check if corosync ring IPs are configured on the current nodes interfaces - my $check_ip = sub { - my $ip = shift; - if (defined($ip)) { - if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) { - my $host = $ip; - eval { $ip = PVE::Network::get_ip_from_hostname($host); }; - if ($@) { - &$error("cannot use '$host': $@\n") ; - return; - } - } - - my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32"; - my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr); - - &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n") - if (scalar(@$configured_ips) != 1); - } - }; - - &$check_ip($param->{ring0_addr}); - &$check_ip($param->{ring1_addr}); - - warn "warning, ignore the following errors:\n$warnings" if $warnings; - die "detected the following error(s):\n$errors" if $errors; - # make sure known_hosts is on local filesystem PVE::Cluster::ssh_unmerge_known_hosts(); @@ -372,11 +279,8 @@ __PACKAGE__->register_method ({ 'pvecm', 'addnode', $nodename, '--force', 1]; push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid}; - push @$cmd, '--votes', $param->{votes} if defined($param->{votes}); - push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr}); - push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr}); if (system (@$cmd) != 0) { @@ -394,58 +298,10 @@ __PACKAGE__->register_method ({ system(@$cmd) == 0 || die "can't rsync data from host '$host'\n"; - mkdir "/etc/corosync"; - my $confbase = basename($clusterconf); + my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf"); + my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey"); - $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'"; - system($cmd) == 0 || die "can't copy cluster configuration\n"; - - my $keybase = basename($authfile); - system ("cp '$tmpdir/$keybase' '$authfile'") == 0 || - die "can't copy '$tmpdir/$keybase' to '$authfile'\n"; - - print "stopping pve-cluster service\n"; - - system("umount $basedir -f >/dev/null 2>&1"); - system("systemctl stop pve-cluster") == 0 || - die "can't stop pve-cluster service\n"; - - backup_database(); - - unlink $dbfile; - - system("systemctl start pve-cluster") == 0 || - die "starting pve-cluster failed\n"; - - system("systemctl start corosync"); - - # wait for quorum - my $printqmsg = 1; - while (!PVE::Cluster::check_cfs_quorum(1)) { - if ($printqmsg) { - print "waiting for quorum..."; - STDOUT->flush(); - $printqmsg = 0; - } - sleep(1); - } - print "OK\n" if !$printqmsg; - - my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); - - print "generating node certificates\n"; - PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); - - print "merge known_hosts file\n"; - PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); - - print "restart services\n"; - # restart pvedaemon (changed certs) - system("systemctl restart pvedaemon"); - # restart pveproxy (changed certs) - system("systemctl restart pveproxy"); - - print "successfully added node '$nodename' to cluster.\n"; + PVE::Cluster::finish_join($host, $corosync_conf, $corosync_authkey); }; my $err = $@; diff --git a/data/PVE/Cluster.pm b/data/PVE/Cluster.pm index 5fc7d86..b8dfe5e 100644 --- a/data/PVE/Cluster.pm +++ b/data/PVE/Cluster.pm @@ -37,6 +37,14 @@ my $basedir = "/etc/pve"; my $authdir = "$basedir/priv"; my $lockdir = "/etc/pve/priv/lock"; +# cfs and corosync files +my $dbfile = "/var/lib/pve-cluster/config.db"; +my $dbbackupdir = "/var/lib/pve-cluster/backup"; +my $localclusterdir = "/etc/corosync"; +my $localclusterconf = "$localclusterdir/corosync.conf"; +my $authfile = "$localclusterdir/authkey"; +my $clusterconf = "$basedir/corosync.conf"; + my $authprivkeyfn = "$authdir/authkey.key"; my $authpubkeyfn = "$basedir/authkey.pub"; my $pveca_key_fn = "$authdir/pve-root-ca.key"; @@ -1683,4 +1691,142 @@ sub ssh_info_to_command { return $cmd; } +sub assert_joinable { + my ($ring0_addr, $ring1_addr, $force) = @_; + + my ($errors, $warnings) = ('', ''); + my $error = sub { + my ($msg, $suppress) = @_; + + if ($suppress) { + $warnings .= "* $msg\n"; + } else { + $errors .= "* $msg\n"; + } + }; + + if (!$param->{force}) { + + if (-f $authfile) { + $error->("authentication key '$authfile' already exists", $force); + } + + if (-f $clusterconf) { + $error->("cluster config '$clusterconf' already exists", $force); + } + + my $vmlist = PVE::Cluster::get_vmlist(); + if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) { + $error->("this host already contains virtual guests", $force); + } + + if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) { + $error->("corosync is already running, is this node already in a cluster?!", $force); + } + } + + # check if corosync ring IPs are configured on the current nodes interfaces + my $check_ip = sub { + my $ip = shift // return; + if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) { + my $host = $ip; + eval { $ip = PVE::Network::get_ip_from_hostname($host); }; + if ($@) { + $error->("cannot use '$host': $@\n", 1) ; + return; + } + } + + my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32"; + my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr); + + $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n") + if (scalar(@$configured_ips) != 1); + }; + + $check_ip->($ring0_addr); + $check_ip->($ring1_addr); + + warn "warning, ignore the following errors:\n$warnings" if $warnings; + die "detected the following error(s):\n$errors" if $errors; +} + +my $backup_cfs_database = sub { + my ($dbfile) = @_; + + mkdir $dbbackupdir; + + print "backup old database\n"; + my $ctime = time(); + my $cmd = [ + ['echo', '.dump'], + ['sqlite3', $dbfile], + ['gzip', '-', \ ">${dbbackupdir}/config-${ctime}.sql.gz"], + ]; + + PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n"); + + # purge older backup + my $maxfiles = 10; + my @bklist = (); + foreach my $fn (<$dbbackupdir/config-*.sql.gz>) { + if ($fn =~ m!/config-(\d+)\.sql.gz$!) { + push @bklist, [$fn, $1]; + } + } + + @bklist = sort { $b->[1] <=> $a->[1] } @bklist; + while (scalar (@bklist) >= $maxfiles) { + my $d = pop @bklist; + print "delete old backup '$d->[0]'\n"; + unlink $d->[0]; + } +}; + +sub finish_join { + my ($nodename, $corosync_conf, $corosync_authkey) = @_; + + mkdir "$localclusterdir"; + PVE::Tools::file_set_contents($authfile, $corosync_authkey); + PVE::Tools::file_set_contents($localclusterconf, $corosync_conf); + + print "stopping pve-cluster service\n"; + + system("umount $basedir -f >/dev/null 2>&1"); + die "can't stop pve-cluster service\n" if system("systemctl stop pve-cluster") != 0; + + $backup_cfs_database->($dbfile); + unlink $dbfile; + + system("systemctl start pve-cluster") == 0 || die "starting pve-cluster failed\n"; + system("systemctl start corosync"); + + # wait for quorum + my $printqmsg = 1; + while (!PVE::Cluster::check_cfs_quorum(1)) { + if ($printqmsg) { + print "waiting for quorum..."; + STDOUT->flush(); + $printqmsg = 0; + } + sleep(1); + } + print "OK\n" if !$printqmsg; + + my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); + + print "generating node certificates\n"; + PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address); + + print "merge known_hosts file\n"; + PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1); + + print "restart services\n"; + # restart pvedaemon and pveproxy (changed certs) + system("systemctl restart pvedaemon pveproxy"); + + print "successfully added node '$nodename' to cluster.\n"; +} + + 1; -- 2.11.0 _______________________________________________ pve-devel mailing list [email protected] https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
