package PVE::CLI::pvecm; use strict; use warnings; use Cwd qw(getcwd); use File::Path; use File::Basename; use Time::HiRes qw(usleep); use PVE::Tools qw(run_command); use PVE::Cluster; use PVE::INotify; use PVE::JSONSchema qw(get_standard_option); use PVE::RPCEnvironment; use PVE::CLIHandler; use PVE::PTY; use PVE::API2::ClusterConfig; use PVE::Corosync; use PVE::Cluster::Setup; use PVE::SSHInfo; use base qw(PVE::CLIHandler); $ENV{HOME} = '/root'; # for ssh-copy-id my $basedir = "/etc/pve"; my $clusterconf = "$basedir/corosync.conf"; my $libdir = "/var/lib/pve-cluster"; my $authfile = "/etc/corosync/authkey"; sub setup_environment { PVE::RPCEnvironment->setup_default_cli_env(); } __PACKAGE__->register_method({ name => 'keygen', path => 'keygen', method => 'PUT', description => "Generate new cryptographic key for corosync.", parameters => { additionalProperties => 0, properties => { filename => { type => 'string', description => "Output file name", }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $filename = $param->{filename}; # test EUID $> == 0 || die "Error: Authorization key must be generated as root user.\n"; my $dirname = dirname($filename); die "key file '$filename' already exists\n" if -e $filename; File::Path::make_path($dirname) if $dirname; run_command(['corosync-keygen', '-l', '-k', $filename]); return undef; }, }); my $foreach_member = sub { my ($code, $noerr) = @_; my $members = PVE::Cluster::get_members(); foreach my $node (sort keys %$members) { if (my $ip = $members->{$node}->{ip}) { $code->($node, $ip); } else { die "cannot get the cluster IP for node '$node'.\n" if !$noerr; warn "cannot get the cluster IP for node '$node'.\n"; return undef; } } }; __PACKAGE__->register_method({ name => 'setup_qdevice', path => 'setup_qdevice', method => 'PUT', description => "Setup the use of a QDevice", parameters => { additionalProperties => 0, properties => { address => { type => 'string', format => 'ip', description => "Specifies the network address of an external corosync QDevice", }, network => { type => 'string', format => 'CIDR', description => 'The network which should be used to connect to the external qdevice', optional => 1, }, force => { type => 'boolean', description => "Do not throw error on possible dangerous operations.", optional => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Corosync::check_conf_exists(); my $members = PVE::Cluster::get_members(); foreach my $node (sort keys %$members) { die "All nodes must be online! Node $node is offline, aborting.\n" if !$members->{$node}->{online}; } my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); die "QDevice already configured!\n" if defined($conf->{main}->{quorum}->{device}) && !$param->{force}; my $network = $param->{network}; my $model = "net"; my $algorithm = 'ffsplit'; if (scalar(%{$members}) & 1) { if ($param->{force}) { $algorithm = 'lms'; } else { die "Clusters with an odd node count are not officially supported!\n"; } } my $qnetd_addr = $param->{address}; my $base_dir = "/etc/corosync/qdevice/net"; my $db_dir_qnetd = "/etc/corosync/qnetd/nssdb"; my $db_dir_node = "$base_dir/nssdb"; my $ca_export_base = "qnetd-cacert.crt"; my $ca_export_file = "$db_dir_qnetd/$ca_export_base"; my $crq_file_base = "qdevice-net-node.crq"; my $p12_file_base = "qdevice-net-node.p12"; my $qdevice_certutil = "corosync-qdevice-net-certutil"; my $qnetd_certutil = "corosync-qnetd-certutil"; my $clustername = $conf->{main}->{totem}->{cluster_name}; run_command(['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$qnetd_addr"]); if (-d $db_dir_node) { # FIXME: check on all nodes?! if ($param->{force}) { rmtree $db_dir_node; } else { die "QDevice certificate store already initialised, set force to delete!\n"; } } my $ssh_cmd = ['ssh', '-o', 'BatchMode=yes', '-lroot']; my $scp_cmd = ['scp', '-o', 'BatchMode=yes']; print "\nINFO: initializing qnetd server\n"; run_command( [@$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-i"], noerr => 1, ); print "\nINFO: copying CA cert and initializing on all nodes\n"; run_command( [@$scp_cmd, "root\@\[$qnetd_addr\]:$ca_export_file", "/etc/pve/$ca_export_base"]); $foreach_member->(sub { my ($node, $ip) = @_; my $ssh_cluster_cmd = PVE::SSHInfo::ssh_info_to_command({ ip => $ip, name => $node }); my $outsub = sub { print "\nnode '$node': " . shift }; run_command( [ @$ssh_cluster_cmd, '--', $qdevice_certutil, "-i", "-c", "/etc/pve/$ca_export_base", ], noerr => 1, outfunc => $outsub, ); }); unlink "/etc/pve/$ca_export_base"; print "\nINFO: generating cert request\n"; run_command([$qdevice_certutil, "-r", "-n", $clustername]); print "\nINFO: copying exported cert request to qnetd server\n"; run_command([@$scp_cmd, "$db_dir_node/$crq_file_base", "root\@\[$qnetd_addr\]:/tmp"]); print "\nINFO: sign and export cluster cert\n"; run_command([ @$ssh_cmd, $qnetd_addr, $qnetd_certutil, "-s", "-c", "/tmp/$crq_file_base", "-n", "$clustername", ]); print "\nINFO: copy exported CRT\n"; run_command([ @$scp_cmd, "root\@\[$qnetd_addr\]:$db_dir_qnetd/cluster-$clustername.crt", "$db_dir_node", ]); print "\nINFO: import certificate\n"; run_command(["$qdevice_certutil", "-M", "-c", "$db_dir_node/cluster-$clustername.crt"]); print "\nINFO: copy and import pk12 cert to all nodes\n"; run_command([@$scp_cmd, "$db_dir_node/$p12_file_base", "/etc/pve/"]); $foreach_member->(sub { my ($node, $ip) = @_; my $ssh_cluster_cmd = PVE::SSHInfo::ssh_info_to_command({ ip => $ip, name => $node }); my $outsub = sub { print "\nnode '$node': " . shift }; run_command( [ @$ssh_cluster_cmd, '--', "$qdevice_certutil", "-m", "-c", "/etc/pve/$p12_file_base", ], outfunc => $outsub, ); }); unlink "/etc/pve/$p12_file_base"; my $code = sub { my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); my $quorum_section = $conf->{main}->{quorum}; die "Qdevice already configured, must be removed before setting up new one!\n" if defined($quorum_section->{device}); # must not be forced! my $qdev_section = { model => $model, "$model" => { tls => 'on', host => $qnetd_addr, algorithm => $algorithm, }, }; $qdev_section->{votes} = 1 if $algorithm eq 'ffsplit'; $quorum_section->{device} = $qdev_section; PVE::Corosync::atomic_write_conf($conf); }; print "\nINFO: add QDevice to cluster configuration\n"; PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code); die $@ if $@; $foreach_member->(sub { my ($node, $ip) = @_; my $ssh_cluster_cmd = PVE::SSHInfo::ssh_info_to_command({ ip => $ip, name => $node }); my $outsub = sub { print "\nnode '$node': " . shift }; print "\nINFO: start and enable corosync qdevice daemon on node '$node'...\n"; run_command( [@$ssh_cluster_cmd, '--', 'systemctl', 'start', 'corosync-qdevice'], outfunc => $outsub, ); run_command( [@$ssh_cluster_cmd, '--', 'systemctl', 'enable', 'corosync-qdevice'], outfunc => $outsub, ); }); run_command(['corosync-cfgtool', '-R']); # do cluster wide config reload return undef; }, }); __PACKAGE__->register_method({ name => 'remove_qdevice', path => 'remove_qdevice', method => 'DELETE', description => "Remove a configured QDevice", parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Corosync::check_conf_exists(); my $members = PVE::Cluster::get_members(); foreach my $node (sort keys %$members) { die "All nodes must be online! Node $node is offline, aborting.\n" if !$members->{$node}->{online}; } my $code = sub { my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); my $quorum_section = $conf->{main}->{quorum}; die "No QDevice configured!\n" if !defined($quorum_section->{device}); delete $quorum_section->{device}; PVE::Corosync::atomic_write_conf($conf); # cleanup qdev state (cert storage) my $qdev_state_dir = "/etc/corosync/qdevice"; $foreach_member->(sub { my ($node, $ip) = @_; my $ssh_cluster_cmd = PVE::SSHInfo::ssh_info_to_command({ ip => $ip, name => $node }); run_command([@$ssh_cluster_cmd, '--', 'rm', '-rf', $qdev_state_dir]); }); }; PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code); die $@ if $@; $foreach_member->(sub { my ($node, $ip) = @_; my $ssh_cluster_cmd = PVE::SSHInfo::ssh_info_to_command({ ip => $ip, name => $node }); run_command([@$ssh_cluster_cmd, '--', 'systemctl', 'stop', 'corosync-qdevice']); run_command([@$ssh_cluster_cmd, '--', 'systemctl', 'disable', 'corosync-qdevice']); }); run_command(['corosync-cfgtool', '-R']); print "\nRemoved Qdevice.\n"; return undef; }, }); __PACKAGE__->register_method({ name => 'add', path => 'add', method => 'PUT', description => "Adds the current node to an existing cluster.", parameters => { additionalProperties => 0, properties => PVE::Corosync::add_corosync_link_properties({ hostname => { type => 'string', description => "Hostname (or IP) of an existing cluster member.", }, nodeid => get_standard_option('corosync-nodeid'), votes => { type => 'integer', description => "Number of votes for this node", minimum => 0, optional => 1, }, force => { type => 'boolean', description => "Do not throw error if node already exists.", optional => 1, }, fingerprint => get_standard_option('fingerprint-sha256', { optional => 1, }), 'use_ssh' => { type => 'boolean', description => "Always use SSH to join, even if peer may do it over API.", optional => 1, }, }), }, returns => { type => 'null' }, code => sub { my ($param) = @_; # avoid "transport endpoint not connected" errors that occur if # restarting pmxcfs while in fuse-mounted /etc/pve die "Navigate out of $basedir before running 'pvecm add', for example by running 'cd'.\n" if getcwd() =~ m!^$basedir(/.*)?$!; my $nodename = PVE::INotify::nodename(); my $host = $param->{hostname}; my $worker = sub { if (!$param->{use_ssh}) { my $password = PVE::PTY::read_password("Please enter superuser (root) password for '$host': "); delete $param->{use_ssh}; $param->{password} = $password; my $local_cluster_lock = "/var/lock/pvecm.lock"; PVE::Tools::lock_file( $local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param, ); if (my $err = $@) { if ( ref($err) eq 'PVE::APIClient::Exception' && defined($err->{code}) && $err->{code} == 501 ) { $err = "Remote side is not able to use API for Cluster join!\n" . "Pass the 'use_ssh' switch or update the remote side.\n"; } die $err; } return; # all OK, the API join endpoint successfully set us up } # allow fallback to old ssh only join if wished or needed my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); my $links = PVE::Corosync::extract_corosync_link_args($param); PVE::Cluster::Setup::assert_joinable($local_ip_address, $links, $param->{force}); PVE::Cluster::Setup::setup_sshd_config(); PVE::Cluster::Setup::setup_rootsshconfig(); PVE::Cluster::Setup::setup_ssh_keys(); # make sure known_hosts is on local filesystem PVE::Cluster::Setup::ssh_unmerge_known_hosts(); my $cmd = ['ssh-copy-id', '-i', '/root/.ssh/id_rsa', "root\@$host"]; run_command( $cmd, 'outfunc' => sub { }, 'errfunc' => sub { }, 'errmsg' => "unable to copy ssh ID", ); $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'apiver']; my $remote_apiver = 0; run_command( $cmd, 'outfunc' => sub { $remote_apiver = shift; chomp $remote_apiver; }, 'noerr' => 1, ); PVE::Cluster::Setup::assert_we_can_join_cluster_version($remote_apiver); $cmd = ['ssh', $host, '-o', 'BatchMode=yes', 'pvecm', 'addnode', $nodename, '--force', 1]; push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid}; push @$cmd, '--votes', $param->{votes} if defined($param->{votes}); my $link_desc = get_standard_option('corosync-link'); foreach my $link (keys %$links) { push @$cmd, "--link$link", PVE::JSONSchema::print_property_string($links->{$link}, $link_desc->{format}); } # this will be used as fallback if no links are specified if (!%$links) { push @$cmd, '--link0', $local_ip_address if $remote_apiver == 0; push @$cmd, '--new_node_ip', $local_ip_address if $remote_apiver >= 1; print "No cluster network links passed explicitly, fallback to local node" . " IP '$local_ip_address'\n"; } if (system(@$cmd) != 0) { my $cmdtxt = join(' ', @$cmd); die "unable to add node: command failed ($cmdtxt)\n"; } my $tmpdir = "$libdir/.pvecm_add.tmp.$$"; mkdir $tmpdir; eval { print "copy corosync auth key\n"; $cmd = [ 'rsync', '--rsh=ssh -l root -o BatchMode=yes', '-lpgoq', "[$host]:$authfile", "[$host]:$clusterconf", $tmpdir, ]; system(@$cmd) == 0 || die "can't rsync data from host '$host'\n"; my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf"); my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey"); PVE::Cluster::Setup::finish_join($nodename, $corosync_conf, $corosync_authkey); }; my $err = $@; rmtree $tmpdir; die $err if $err; }; # use a synced worker so we get a nice task log when joining through CLI my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); $rpcenv->fork_worker('clusterjoin', '', $authuser, $worker); return undef; }, }); __PACKAGE__->register_method({ name => 'status', path => 'status', method => 'GET', description => "Displays the local view of the cluster status.", parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Corosync::check_conf_exists(); my $conf = eval { PVE::Cluster::cfs_read_file("corosync.conf") } // {}; warn "$@" if $@; my $totem = PVE::Corosync::totem_config($conf); if (scalar(%$totem)) { my $print_info = sub { my ($label, $key, $default) = @_; my $val = $totem->{$key} // $default; printf "%-17s %s\n", "$label:", "$val"; }; printf "Cluster information\n"; printf "-------------------\n"; $print_info->('Name', 'cluster_name', 'UNKOWN?'); $print_info->('Config Version', 'config_version', -1); $print_info->('Transport', 'transport', 'knet'); $print_info->('Secure auth', 'secauth', 'off'); printf "\n"; } exec('corosync-quorumtool', '-siH'); exit(-1); # should not be reached }, }); __PACKAGE__->register_method({ name => 'nodes', path => 'nodes', method => 'GET', description => "Displays the local view of the cluster nodes.", parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Corosync::check_conf_exists(); exec('corosync-quorumtool', '-l'); exit(-1); # should not be reached }, }); __PACKAGE__->register_method({ name => 'expected', path => 'expected', method => 'PUT', description => "Tells corosync a new value of expected votes.", parameters => { additionalProperties => 0, properties => { expected => { type => 'integer', description => "Expected votes", minimum => 1, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Corosync::check_conf_exists(); exec('corosync-quorumtool', '-e', $param->{expected}); exit(-1); # should not be reached }, }); __PACKAGE__->register_method({ name => 'updatecerts', path => 'updatecerts', method => 'PUT', description => "Update node certificates (and generate all needed files/directories).", parameters => { additionalProperties => 0, properties => { force => { description => "Force generation of new SSL certificate.", type => 'boolean', optional => 1, }, silent => { description => "Ignore errors (i.e. when cluster has no quorum).", type => 'boolean', optional => 1, }, 'unmerge-known-hosts' => { description => "Unmerge legacy SSH known hosts.", type => 'boolean', optional => 1, default => 0, }, }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my ($force_new_cert, $silent, $unmerge) = $param->@{qw(force silent unmerge-known-hosts)}; # pveproxy's ExecStartPre calls this, and as we do IO (on /etc/pve) that can hang # (uninterruptible D state) we could fail the whole service, rendering the API guaranteed # inaccessible. Let's rather fail small(er) as the API could still work without this.. my ($_res, $got_timeout) = PVE::Tools::run_fork_with_timeout( 80, # cope with slow HW with headroom for manager service 90s start timeout (bz#5510) sub { # the forked child exits without flushing, which loses buffered output to a pipe # or the journal $| = 1; PVE::Cluster::Setup::generate_local_files(); for (my $i = 0; !PVE::Cluster::check_cfs_quorum(1); $i++) { print "waiting for pmxcfs mount to appear and get quorate...\n" if !$silent && $i % 50 == 0; usleep(100 * 1000); } PVE::Cluster::Setup::updatecerts_and_ssh($force_new_cert, $silent, $unmerge); PVE::Cluster::prepare_observed_file_basedirs(); }, ); if ($got_timeout) { my $msg = "got timeout when trying to ensure cluster certificates and base file" . " hierarchy is set up - no quorum (yet) or hung pmxcfs?\n"; die $msg if !$silent; # this might be unexpected for the $silent case, but in our ExecStartPre use case it's # really better than keeping the user completely in the dark, so maybe split/fix params warn $msg; } return undef; }, }); __PACKAGE__->register_method({ name => 'rotate_ca', path => 'rotate_ca', method => 'POST', description => "Replace the cluster CA with a new key and certificate, for example after the key" . " got exposed, and issue new node certificates on all online nodes.", parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'null' }, code => sub { my ($param) = @_; PVE::Cluster::check_cfs_quorum(); print "generating new cluster CA\n"; PVE::Cluster::Setup::rotate_pveca(); PVE::Cluster::cfs_update(1); my $members = PVE::Cluster::get_members() // {}; my $local_node = PVE::INotify::nodename(); my $nodes = [$local_node, grep { $_ ne $local_node } sort keys %$members]; # every node issues its own certificate, as only it knows all of its names my $failed = []; for my $node (@$nodes) { my $status = eval { PVE::Cluster::Setup::pve_ssl_cert_status($node) }; if (my $err = $@) { warn "checking node certificate on '$node' failed - $err"; push @$failed, $node; next; } my $issued_by = $status ? $status->{'issued-by'} : ''; if ($issued_by eq 'current-ca') { print "node certificate on '$node' is from the new CA already\n"; next; } elsif ($issued_by eq 'other' && $status->{notafter} > time()) { print "keeping node certificate on '$node', as no cluster CA issued it\n"; next; } print "issuing new node certificate on '$node'\n"; eval { my $ssh = []; if ($node ne $local_node) { die "node is offline\n" if !$members->{$node}->{online}; $ssh = PVE::SSHInfo::ssh_info_to_command(PVE::SSHInfo::get_ssh_info($node)); } # SSH can hang before updatecerts starts its own timeout my $cmd = [@$ssh, 'pvecm', 'updatecerts', '--force']; run_command($cmd, outfunc => sub { }, timeout => 120); run_command( [@$ssh, 'systemctl', 'reload-or-restart', 'pveproxy'], timeout => 60, ); }; if (my $err = $@) { warn "issuing new node certificate on '$node' failed - $err"; push @$failed, $node; } } print "Replace the previous cluster CA with the new /etc/pve/pve-root-ca.pem on all" . " clients that trust it. SPICE consoles of running VMs fail until the VM is" . " restarted or migrated.\n"; if (scalar(@$failed)) { my $failed_list = join(', ', @$failed); die "Run 'pvecm updatecerts --force' and 'systemctl reload-or-restart pveproxy' on" . " $failed_list. Up-to-date nodes also do so on their own with the next start of" . " pveproxy or the daily update.\n"; } return undef; }, }); __PACKAGE__->register_method({ name => 'mtunnel', path => 'mtunnel', method => 'POST', description => "Used by VM/CT migration - do not use manually.", parameters => { additionalProperties => 0, properties => { get_migration_ip => { type => 'boolean', default => 0, description => 'return the migration IP, if configured', optional => 1, }, migration_network => { type => 'string', format => 'CIDR', description => 'the migration network used to detect the local migration IP', optional => 1, }, 'run-command' => { type => 'boolean', description => 'Run a command with a tcp socket as standard input.' . ' The IP address and port are printed via this' . " command's stdandard output first, each on a separate line.", optional => 1, }, 'extra-args' => PVE::JSONSchema::get_standard_option('extra-args'), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; if (!PVE::Cluster::check_cfs_quorum(1)) { print "no quorum\n"; return undef; } my $get_local_migration_ip = sub { my ($cidr) = @_; if (!defined($cidr)) { my $dc_conf = cfs_read_file('datacenter.cfg'); $cidr = $dc_conf->{migration}->{network} if defined($dc_conf->{migration}->{network}); } if (defined($cidr)) { my $ips = PVE::Network::get_local_ip_from_cidr($cidr); die "could not get migration ip: no IP address configured on local " . "node for network '$cidr'\n" if scalar(@$ips) == 0; die "could not get migration ip: multiple, different, IP address configured for " . "network '$cidr'\n" if scalar(@$ips) > 1 && grep { @$ips[0] ne $_ } @$ips; return @$ips[0]; } return undef; }; my $network = $param->{migration_network}; if ($param->{get_migration_ip}) { die "cannot use --run-command with --get_migration_ip\n" if $param->{'run-command'}; if (my $ip = $get_local_migration_ip->($network)) { print "ip: '$ip'\n"; } else { print "no ip\n"; } # do not keep tunnel open when asked for migration ip return undef; } if ($param->{'run-command'}) { my $cmd = $param->{'extra-args'}; die "missing command\n" if !$cmd || !scalar(@$cmd); # Get an ip address to listen on, and find a free migration port my ($ip, $family); if (defined($network)) { $ip = $get_local_migration_ip->($network) or die "failed to get migration IP address to listen on\n"; $family = PVE::Tools::get_host_address_family($ip); } else { my $nodename = PVE::INotify::nodename(); ($ip, $family) = PVE::Network::get_ip_from_hostname($nodename, 0); } my $port = PVE::Tools::next_migrate_port($family, $ip); PVE::Tools::pipe_socket_to_command($cmd, $ip, $port); return undef; } print "tunnel online\n"; *STDOUT->flush(); while (my $line = ) { chomp $line; last if $line =~ m/^quit$/; } return undef; }, }); our $cmddef = { apiver => [ 'PVE::API2::ClusterConfig', 'join_api_version', [], {}, sub { my $apiver = shift; print "$apiver\n"; }, ], keygen => [__PACKAGE__, 'keygen', ['filename']], create => ['PVE::API2::ClusterConfig', 'create', ['clustername']], add => [__PACKAGE__, 'add', ['hostname']], addnode => ['PVE::API2::ClusterConfig', 'addnode', ['node']], delnode => ['PVE::API2::ClusterConfig', 'delnode', ['node']], status => [__PACKAGE__, 'status'], nodes => [__PACKAGE__, 'nodes'], expected => [__PACKAGE__, 'expected', ['expected']], updatecerts => [__PACKAGE__, 'updatecerts', []], mtunnel => [__PACKAGE__, 'mtunnel', ['extra-args']], ca => { rotate => [__PACKAGE__, 'rotate_ca', []], }, qdevice => { setup => [__PACKAGE__, 'setup_qdevice', ['address']], remove => [__PACKAGE__, 'remove_qdevice', []], }, }; 1;