package PVE::API2::ClusterConfig; use strict; use warnings; use PVE::Exception; use PVE::Tools; use PVE::SafeSyslog; use PVE::RESTHandler; use PVE::RPCEnvironment; use PVE::JSONSchema qw(get_standard_option); use PVE::Cluster; use PVE::APIClient::LWP; use PVE::Corosync; use PVE::Cluster::Setup; use IO::Socket::UNIX; use base qw(PVE::RESTHandler); my $clusterconf = "/etc/pve/corosync.conf"; my $authfile = "/etc/corosync/authkey"; my $local_cluster_lock = "/var/lock/pvecm.lock"; my $nodeid_desc = { type => 'integer', description => "Node id for this node.", minimum => 1, optional => 1, }; PVE::JSONSchema::register_standard_option("corosync-nodeid", $nodeid_desc); __PACKAGE__->register_method({ name => 'index', path => '', method => 'GET', description => "Directory index.", permissions => { check => ['perm', '/', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'array', items => { type => "object", properties => {}, }, links => [{ rel => 'child', href => "{name}" }], }, code => sub { my ($param) = @_; my $result = [ { name => 'nodes' }, { name => 'totem' }, { name => 'join' }, { name => 'qdevice' }, { name => 'apiversion' }, ]; return $result; }, }); __PACKAGE__->register_method({ name => 'join_api_version', path => 'apiversion', method => 'GET', description => "Return the version of the cluster join API available on this node.", permissions => { check => ['perm', '/', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'integer', minimum => 0, description => "Cluster Join API version, currently " . PVE::Cluster::Setup::JOIN_API_VERSION, }, code => sub { return PVE::Cluster::Setup::JOIN_API_VERSION; }, }); __PACKAGE__->register_method({ name => 'create', path => '', method => 'POST', protected => 1, description => "Generate new cluster configuration. If no links given," . " default to local IP address as link0.", parameters => { additionalProperties => 0, properties => PVE::Corosync::add_corosync_link_properties({ clustername => { description => "The name of the cluster.", type => 'string', format => 'pve-node', maxLength => 15, }, nodeid => get_standard_option('corosync-nodeid'), votes => { type => 'integer', description => "Number of votes for this node.", minimum => 1, optional => 1, }, 'token-coefficient' => { type => 'integer', description => "Coefficient used to determine Corosync's token timeout. See the" . " corosync.conf(5) manual for more details.", default => 125, minimum => 0, optional => 1, }, }), }, returns => { type => 'string' }, code => sub { my ($param) = @_; $param->{'token-coefficient'} //= 125; die "cluster config '$clusterconf' already exists\n" if -f $clusterconf; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $code = sub { STDOUT->autoflush(); PVE::Cluster::Setup::setup_sshd_config(1); PVE::Cluster::Setup::setup_rootsshconfig(); PVE::Cluster::Setup::setup_ssh_keys(); PVE::Tools::run_command(['/usr/sbin/corosync-keygen', '-lk', $authfile]) if !-f $authfile; die "no authentication key available\n" if -f !$authfile; my $nodename = PVE::INotify::nodename(); # get the corosync basis config for the new cluster my $config = PVE::Corosync::create_conf($nodename, $param); print "Writing corosync config to /etc/pve/corosync.conf\n"; PVE::Corosync::atomic_write_conf($config); my $local_ip_address = PVE::Cluster::remote_node_ip($nodename); PVE::Cluster::Setup::ssh_merge_keys(); PVE::Cluster::Setup::gen_pve_node_files($nodename, $local_ip_address); PVE::Cluster::Setup::ssh_merge_known_hosts($nodename, $local_ip_address, 1); print "Restart corosync and cluster filesystem\n"; PVE::Tools::run_command('systemctl restart corosync pve-cluster'); }; my $worker = sub { PVE::Tools::lock_file($local_cluster_lock, 10, $code); die $@ if $@; }; return $rpcenv->fork_worker('clustercreate', $param->{clustername}, $authuser, $worker); }, }); __PACKAGE__->register_method({ name => 'nodes', path => 'nodes', method => 'GET', description => "Corosync node list.", permissions => { check => ['perm', '/', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => {}, }, returns => { type => 'array', items => { type => "object", properties => { node => { type => 'string' }, }, }, links => [{ rel => 'child', href => "{node}" }], }, code => sub { my ($param) = @_; my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); my $nodelist = PVE::Corosync::nodelist($conf); return PVE::RESTHandler::hash_to_array($nodelist, 'node'); }, }); # lock method to ensure local and cluster wide atomicity # if we're a single node cluster just lock locally, we have no other cluster # node which we could contend with, else also acquire a cluster wide lock my $config_change_lock = sub { my ($code) = @_; PVE::Tools::lock_file( $local_cluster_lock, 10, sub { PVE::Cluster::cfs_update(1); my $members = PVE::Cluster::get_members(); if (scalar(keys %$members) > 1) { my $res = PVE::Cluster::cfs_lock_file('corosync.conf', 10, $code); # cfs_lock_file only sets $@ but lock_file doesn't propagates $@ unless we die here die $@ if defined($@); return $res; } else { return $code->(); } }, ); }; __PACKAGE__->register_method({ name => 'addnode', path => 'nodes/{node}', method => 'POST', protected => 1, description => "Adds a node to the cluster configuration. This call is for internal use.", parameters => { additionalProperties => 0, properties => PVE::Corosync::add_corosync_link_properties({ node => get_standard_option('pve-node'), nodeid => get_standard_option('corosync-nodeid'), votes => { type => 'integer', description => "Number of votes for this node", minimum => 0, optional => 1, }, force => { type => 'boolean', description => "Do not throw error if node already exists.", optional => 1, }, new_node_ip => { type => 'string', description => "IP Address of node to add. Used as fallback if no links are given.", format => 'ip', optional => 1, }, apiversion => { type => 'integer', description => 'The JOIN_API_VERSION of the new node.', optional => 1, }, }), }, returns => { type => "object", properties => { corosync_authkey => { type => 'string', }, corosync_conf => { type => 'string', }, warnings => { type => 'array', items => { type => 'string', }, }, }, }, code => sub { my ($param) = @_; $param->{apiversion} //= 0; PVE::Cluster::Setup::assert_node_can_join_our_version($param->{apiversion}); PVE::Cluster::check_cfs_quorum(); my $vc_errors; my $vc_warnings; my $code = sub { my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); my $nodelist = PVE::Corosync::nodelist($conf); my $totem_cfg = PVE::Corosync::totem_config($conf); ($vc_errors, $vc_warnings) = PVE::Corosync::verify_conf($conf); die if scalar(@$vc_errors); my $name = $param->{node}; # ensure we do not reuse an address, that can crash the whole cluster! my $check_duplicate_addr = sub { my $link = shift; return if !defined($link) || !defined($link->{address}); my $addr = $link->{address}; while (my ($k, $v) = each %$nodelist) { next if $k eq $name; # allows re-adding a node if force is set for my $linknumber (0 .. PVE::Corosync::MAX_LINK_INDEX) { my $id = "ring${linknumber}_addr"; next if !defined($v->{$id}); die "corosync: address '$addr' already used on link $id by node '$k'\n" if $v->{$id} eq $addr; } } }; my $links = PVE::Corosync::extract_corosync_link_args($param); foreach my $link (values %$links) { $check_duplicate_addr->($link); } # Make sure that the newly added node has links compatible with the # rest of the cluster. To start, extract all links that currently # exist. Check any node, all have the same links here (because of # verify_conf above). my $node_options = (values %$nodelist)[0]; my $cluster_links = {}; foreach my $opt (keys %$node_options) { my ($linktype, $linkid) = PVE::Corosync::parse_link_entry($opt); next if !defined($linktype); $cluster_links->{$linkid} = $node_options->{$opt}; } # in case no fallback IP was passed, but instead only a single link, # treat it as fallback to allow link-IDs to be matched automatically # FIXME: remove in 8.0 or when joining an old node not supporting # new_node_ip becomes infeasible otherwise my $legacy_fallback = 0; if (!$param->{new_node_ip} && scalar(%$links) == 1 && $param->{apiversion} == 0) { my $passed_link_id = (keys %$links)[0]; my $passed_link = delete $links->{$passed_link_id}; $param->{new_node_ip} = $passed_link->{address}; $legacy_fallback = 1; } if (scalar(%$links)) { # verify specified links exist and none are missing for my $linknum (0 .. PVE::Corosync::MAX_LINK_INDEX) { my $have_cluster_link = defined($cluster_links->{$linknum}); my $have_new_link = defined($links->{$linknum}); die "corosync: link $linknum exists in cluster config but wasn't specified for new node\n" if $have_cluster_link && !$have_new_link; die "corosync: link $linknum specified for new node doesn't exist in cluster config\n" if !$have_cluster_link && $have_new_link; } } else { # when called without any link parameters, fall back to # new_node_ip, if all existing nodes only have a single link too die "no links and no fallback ip (new_node_ip) given, cannot join cluster\n" if !$param->{new_node_ip}; my $cluster_link_count = scalar(%$cluster_links); if ($cluster_link_count == 1) { my $linknum = (keys %$cluster_links)[0]; $links->{$linknum} = { address => $param->{new_node_ip} }; } else { die "cluster has $cluster_link_count links, but only 1 given" if $legacy_fallback; die "no links given but multiple links found on other nodes," . " fallback only supported on single-link clusters\n"; } } if (defined(my $res = $nodelist->{$name})) { $param->{nodeid} = $res->{nodeid} if !$param->{nodeid}; $param->{votes} = $res->{quorum_votes} if !defined($param->{votes}); if ( $res->{quorum_votes} == $param->{votes} && $res->{nodeid} == $param->{nodeid} && $param->{force} ) { print "forcing overwrite of configured node '$name'\n"; } else { die "can't add existing node '$name'\n"; } } elsif (!$param->{nodeid}) { my $nodeid = 1; while (1) { my $found = 0; foreach my $v (values %$nodelist) { if ($v->{nodeid} eq $nodeid) { $found = 1; $nodeid++; last; } } last if !$found; } $param->{nodeid} = $nodeid; } $param->{votes} = 1 if !defined($param->{votes}); PVE::Cluster::Setup::gen_local_dirs($name); eval { PVE::Cluster::Setup::ssh_merge_keys(); }; warn $@ if $@; $nodelist->{$name} = { nodeid => $param->{nodeid}, name => $name, }; $nodelist->{$name}->{quorum_votes} = $param->{votes} if $param->{votes}; foreach my $link (keys %$links) { $nodelist->{$name}->{"ring${link}_addr"} = $links->{$link}->{address}; } PVE::Cluster::log_msg('notice', 'root@pam', "adding node $name to cluster"); PVE::Corosync::update_nodelist($conf, $nodelist); }; $config_change_lock->($code); # If vc_errors is set, we died because of verify_conf. # Raise this error, since it contains more information than just a single-line string. if (defined($vc_errors) && scalar(@$vc_errors)) { my $err_hash = {}; my $add_errs = sub { my ($type, @arr) = @_; return if !scalar(@arr); $err_hash->{"${type}$_"} = $arr[$_] for 0 .. $#arr; }; $add_errs->("warning", @$vc_warnings); $add_errs->("error", @$vc_errors); PVE::Exception::raise("invalid corosync.conf\n", errors => $err_hash); } die $@ if $@; my $res = { corosync_authkey => PVE::Tools::file_get_contents($authfile), corosync_conf => PVE::Tools::file_get_contents($clusterconf), warnings => $vc_warnings, }; return $res; }, }); __PACKAGE__->register_method({ name => 'delnode', path => 'nodes/{node}', method => 'DELETE', protected => 1, description => "Removes a node from the cluster configuration.", parameters => { additionalProperties => 0, properties => { node => get_standard_option('pve-node'), }, }, returns => { type => 'null' }, code => sub { my ($param) = @_; my $local_node = PVE::INotify::nodename(); die "Cannot delete myself from cluster!\n" if $param->{node} eq $local_node; PVE::Cluster::check_cfs_quorum(); my $code = sub { my $conf = PVE::Cluster::cfs_read_file("corosync.conf"); my $nodelist = PVE::Corosync::nodelist($conf); my $node; my $nodeid; foreach my $tmp_node (keys %$nodelist) { my $d = $nodelist->{$tmp_node}; my $ring0_addr = $d->{ring0_addr}; my $ring1_addr = $d->{ring1_addr}; if ( ($tmp_node eq $param->{node}) || (defined($ring0_addr) && ($ring0_addr eq $param->{node})) || (defined($ring1_addr) && ($ring1_addr eq $param->{node})) ) { $node = $tmp_node; $nodeid = $d->{nodeid}; last; } } die "Node/IP: $param->{node} is not a known host of the cluster.\n" if !defined($node); PVE::Cluster::log_msg('notice', 'root@pam', "deleting node $node from cluster"); delete $nodelist->{$node}; # allowed to fail when node is already shut down! eval { PVE::Tools::run_command(['corosync-cfgtool', '-k', $nodeid]) if defined($nodeid); }; PVE::Corosync::update_nodelist($conf, $nodelist); }; $config_change_lock->($code); die $@ if $@; return undef; }, }); __PACKAGE__->register_method({ name => 'join_info', path => 'join', permissions => { check => ['perm', '/', ['Sys.Audit']], }, method => 'GET', description => "Get information needed to join this cluster over the connected node.", parameters => { additionalProperties => 0, properties => { node => get_standard_option( 'pve-node', { description => "The node for which the joinee gets the nodeinfo. ", default => "current connected node", optional => 1, }, ), }, }, returns => { type => 'object', additionalProperties => 0, properties => { nodelist => { type => 'array', items => { type => "object", additionalProperties => 1, properties => { name => get_standard_option('pve-node'), nodeid => get_standard_option('corosync-nodeid'), ring0_addr => get_standard_option('corosync-link'), quorum_votes => { type => 'integer', minimum => 0 }, pve_addr => { type => 'string', format => 'ip' }, pve_fp => get_standard_option('fingerprint-sha256'), }, }, }, preferred_node => get_standard_option('pve-node'), totem => { type => 'object' }, config_digest => { type => 'string' }, }, }, code => sub { my ($param) = @_; my $nodename = $param->{node} // PVE::INotify::nodename(); PVE::Cluster::cfs_update(1); my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); # FIXME: just return undef or empty object in PVE 8.0 (check if manager can handle it!) PVE::Exception::raise( 'node is not in a cluster, no join info available!', code => HTTP::Status::HTTP_FAILED_DEPENDENCY, ) if !($conf && $conf->{main}); my $totem_cfg = $conf->{main}->{totem} // {}; my $nodelist = $conf->{main}->{nodelist}->{node} // {}; my $corosync_config_digest = $conf->{digest}; die "unknown node '$nodename'\n" if !$nodelist->{$nodename}; foreach my $name (keys %$nodelist) { my $node = $nodelist->{$name}; $node->{pve_fp} = PVE::Cluster::get_node_fingerprint($name); $node->{pve_addr} = scalar(PVE::Cluster::remote_node_ip($name)); } my $res = { nodelist => [values %$nodelist], preferred_node => $nodename, totem => $totem_cfg, config_digest => $corosync_config_digest, }; return $res; }, }); __PACKAGE__->register_method({ name => 'join', path => 'join', method => 'POST', protected => 1, description => "Joins this node into an existing cluster. If no links are" . " given, default to IP resolved by node's hostname on single" . " link (fallback fails for clusters with multiple links).", parameters => { additionalProperties => 0, properties => PVE::Corosync::add_corosync_link_properties({ hostname => { type => 'string', description => "Hostname (or IP) of an existing cluster member.", }, nodeid => get_standard_option('corosync-nodeid'), votes => { type => 'integer', description => "Number of votes for this node", minimum => 0, optional => 1, }, force => { type => 'boolean', description => "Do not throw error if node already exists.", optional => 1, }, fingerprint => get_standard_option('fingerprint-sha256'), password => { description => "Superuser (root) password of peer node.", type => 'string', sensitive => 1, maxLength => 128, }, }), }, returns => { type => 'string' }, code => sub { my ($param) = @_; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); my $worker = sub { STDOUT->autoflush(); PVE::Tools::lock_file($local_cluster_lock, 10, \&PVE::Cluster::Setup::join, $param); die $@ if $@; }; return $rpcenv->fork_worker('clusterjoin', undef, $authuser, $worker); }, }); __PACKAGE__->register_method({ name => 'totem', path => 'totem', method => 'GET', description => "Get corosync totem protocol settings.", permissions => { check => ['perm', '/', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => {}, }, returns => { type => "object", properties => {}, }, code => sub { my ($param) = @_; my $conf = PVE::Cluster::cfs_read_file('corosync.conf'); my $totem_cfg = $conf->{main}->{totem}; return $totem_cfg; }, }); __PACKAGE__->register_method({ name => 'status', path => 'qdevice', method => 'GET', protected => 1, description => 'Get QDevice status', permissions => { check => ['perm', '/', ['Sys.Audit']], }, parameters => { additionalProperties => 0, properties => {}, }, returns => { type => "object", }, code => sub { my ($param) = @_; my $result = {}; my $socket_path = "/var/run/corosync-qdevice/corosync-qdevice.sock"; return $result if !-S $socket_path; my $qdevice_socket = IO::Socket::UNIX->new( Type => SOCK_STREAM, Peer => $socket_path, ); print $qdevice_socket "status verbose\n"; my $qdevice_keys = { "Algorithm" => 1, "Echo reply" => 1, "Last poll call" => 1, "Model" => 1, "QNetd host" => 1, "State" => 1, "Tie-breaker" => 1, }; while (my $line = <$qdevice_socket>) { chomp $line; next if $line =~ /^\s/; if ($line =~ /^(.*?)\s*:\s*(.*)$/) { $result->{$1} = $2 if $qdevice_keys->{$1}; } } return $result; }, }); #TODO: possibly add setup and remove methods 1;