[pve-devel] [PATCH cluster v4 03/15] node add: factor out code
Thomas Lamprecht
t.lamprecht at proxmox.com
Tue Jan 9 15:52:51 CET 2018
Factor out common code, which will be used by the new API endpoint to
join a cluster and the old legacy SSH method which we will keep for a
bit.
This is not a completely 1:1 move:
* change &$error(...) to $error->(...)
* removing a few empty lines, where code was so spread out that those
lines resulted in the opposite of what they intended, i.e., less
readability
* rename $backupdir to $dbbackupdir
Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
---
data/PVE/CLI/pvecm.pm | 154 ++------------------------------------------------
data/PVE/Cluster.pm | 146 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 151 insertions(+), 149 deletions(-)
diff --git a/data/PVE/CLI/pvecm.pm b/data/PVE/CLI/pvecm.pm
index a92f831..ca7061f 100755
--- a/data/PVE/CLI/pvecm.pm
+++ b/data/PVE/CLI/pvecm.pm
@@ -21,44 +21,8 @@ $ENV{HOME} = '/root'; # for ssh-copy-id
my $basedir = "/etc/pve";
my $clusterconf = "$basedir/corosync.conf";
my $libdir = "/var/lib/pve-cluster";
-my $backupdir = "/var/lib/pve-cluster/backup";
-my $dbfile = "$libdir/config.db";
my $authfile = "/etc/corosync/authkey";
-sub backup_database {
-
- print "backup old database\n";
-
- mkdir $backupdir;
-
- my $ctime = time();
- my $cmd = [
- ['echo', '.dump'],
- ['sqlite3', $dbfile],
- ['gzip', '-', \ ">${backupdir}/config-${ctime}.sql.gz"],
- ];
-
- run_command($cmd, 'errmsg' => "cannot backup old database\n");
-
- # purge older backup
- my $maxfiles = 10;
-
- my @bklist = ();
- foreach my $fn (<$backupdir/config-*.sql.gz>) {
- if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
- push @bklist, [$fn, $1];
- }
- }
-
- @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
-
- while (scalar (@bklist) >= $maxfiles) {
- my $d = pop @bklist;
- print "delete old backup '$d->[0]'\n";
- unlink $d->[0];
- }
-}
-
__PACKAGE__->register_method ({
name => 'keygen',
@@ -300,67 +264,10 @@ __PACKAGE__->register_method ({
PVE::Cluster::setup_rootsshconfig();
PVE::Cluster::setup_ssh_keys();
+ PVE::Cluster::assert_joinable($param->{ring0_addr}, $param->{ring1_addr}, $param->{force});
+
my $host = $param->{hostname};
- my ($errors, $warnings) = ('', '');
-
- my $error = sub {
- my ($msg, $suppress) = @_;
-
- if ($suppress) {
- $warnings .= "* $msg\n";
- } else {
- $errors .= "* $msg\n";
- }
- };
-
- if (!$param->{force}) {
-
- if (-f $authfile) {
- &$error("authentication key '$authfile' already exists", $param->{force});
- }
-
- if (-f $clusterconf) {
- &$error("cluster config '$clusterconf' already exists", $param->{force});
- }
-
- my $vmlist = PVE::Cluster::get_vmlist();
- if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
- &$error("this host already contains virtual guests", $param->{force});
- }
-
- if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) {
- &$error("corosync is already running, is this node already in a cluster?!", $param->{force});
- }
- }
-
- # check if corosync ring IPs are configured on the current nodes interfaces
- my $check_ip = sub {
- my $ip = shift;
- if (defined($ip)) {
- if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
- my $host = $ip;
- eval { $ip = PVE::Network::get_ip_from_hostname($host); };
- if ($@) {
- &$error("cannot use '$host': $@\n") ;
- return;
- }
- }
-
- my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
- my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
-
- &$error("cannot use IP '$ip', it must be configured exactly once on local node!\n")
- if (scalar(@$configured_ips) != 1);
- }
- };
-
- &$check_ip($param->{ring0_addr});
- &$check_ip($param->{ring1_addr});
-
- warn "warning, ignore the following errors:\n$warnings" if $warnings;
- die "detected the following error(s):\n$errors" if $errors;
-
# make sure known_hosts is on local filesystem
PVE::Cluster::ssh_unmerge_known_hosts();
@@ -372,11 +279,8 @@ __PACKAGE__->register_method ({
'pvecm', 'addnode', $nodename, '--force', 1];
push @$cmd, '--nodeid', $param->{nodeid} if $param->{nodeid};
-
push @$cmd, '--votes', $param->{votes} if defined($param->{votes});
-
push @$cmd, '--ring0_addr', $param->{ring0_addr} if defined($param->{ring0_addr});
-
push @$cmd, '--ring1_addr', $param->{ring1_addr} if defined($param->{ring1_addr});
if (system (@$cmd) != 0) {
@@ -394,58 +298,10 @@ __PACKAGE__->register_method ({
system(@$cmd) == 0 || die "can't rsync data from host '$host'\n";
- mkdir "/etc/corosync";
- my $confbase = basename($clusterconf);
+ my $corosync_conf = PVE::Tools::file_get_contents("$tmpdir/corosync.conf");
+ my $corosync_authkey = PVE::Tools::file_get_contents("$tmpdir/authkey");
- $cmd = "cp '$tmpdir/$confbase' '/etc/corosync/$confbase'";
- system($cmd) == 0 || die "can't copy cluster configuration\n";
-
- my $keybase = basename($authfile);
- system ("cp '$tmpdir/$keybase' '$authfile'") == 0 ||
- die "can't copy '$tmpdir/$keybase' to '$authfile'\n";
-
- print "stopping pve-cluster service\n";
-
- system("umount $basedir -f >/dev/null 2>&1");
- system("systemctl stop pve-cluster") == 0 ||
- die "can't stop pve-cluster service\n";
-
- backup_database();
-
- unlink $dbfile;
-
- system("systemctl start pve-cluster") == 0 ||
- die "starting pve-cluster failed\n";
-
- system("systemctl start corosync");
-
- # wait for quorum
- my $printqmsg = 1;
- while (!PVE::Cluster::check_cfs_quorum(1)) {
- if ($printqmsg) {
- print "waiting for quorum...";
- STDOUT->flush();
- $printqmsg = 0;
- }
- sleep(1);
- }
- print "OK\n" if !$printqmsg;
-
- my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
-
- print "generating node certificates\n";
- PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
-
- print "merge known_hosts file\n";
- PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
-
- print "restart services\n";
- # restart pvedaemon (changed certs)
- system("systemctl restart pvedaemon");
- # restart pveproxy (changed certs)
- system("systemctl restart pveproxy");
-
- print "successfully added node '$nodename' to cluster.\n";
+ PVE::Cluster::finish_join($host, $corosync_conf, $corosync_authkey);
};
my $err = $@;
diff --git a/data/PVE/Cluster.pm b/data/PVE/Cluster.pm
index 5fc7d86..b8dfe5e 100644
--- a/data/PVE/Cluster.pm
+++ b/data/PVE/Cluster.pm
@@ -37,6 +37,14 @@ my $basedir = "/etc/pve";
my $authdir = "$basedir/priv";
my $lockdir = "/etc/pve/priv/lock";
+# cfs and corosync files
+my $dbfile = "/var/lib/pve-cluster/config.db";
+my $dbbackupdir = "/var/lib/pve-cluster/backup";
+my $localclusterdir = "/etc/corosync";
+my $localclusterconf = "$localclusterdir/corosync.conf";
+my $authfile = "$localclusterdir/authkey";
+my $clusterconf = "$basedir/corosync.conf";
+
my $authprivkeyfn = "$authdir/authkey.key";
my $authpubkeyfn = "$basedir/authkey.pub";
my $pveca_key_fn = "$authdir/pve-root-ca.key";
@@ -1683,4 +1691,142 @@ sub ssh_info_to_command {
return $cmd;
}
+sub assert_joinable {
+ my ($ring0_addr, $ring1_addr, $force) = @_;
+
+ my ($errors, $warnings) = ('', '');
+ my $error = sub {
+ my ($msg, $suppress) = @_;
+
+ if ($suppress) {
+ $warnings .= "* $msg\n";
+ } else {
+ $errors .= "* $msg\n";
+ }
+ };
+
+ if (!$param->{force}) {
+
+ if (-f $authfile) {
+ $error->("authentication key '$authfile' already exists", $force);
+ }
+
+ if (-f $clusterconf) {
+ $error->("cluster config '$clusterconf' already exists", $force);
+ }
+
+ my $vmlist = PVE::Cluster::get_vmlist();
+ if ($vmlist && $vmlist->{ids} && scalar(keys %{$vmlist->{ids}})) {
+ $error->("this host already contains virtual guests", $force);
+ }
+
+ if (system("corosync-quorumtool -l >/dev/null 2>&1") == 0) {
+ $error->("corosync is already running, is this node already in a cluster?!", $force);
+ }
+ }
+
+ # check if corosync ring IPs are configured on the current nodes interfaces
+ my $check_ip = sub {
+ my $ip = shift // return;
+ if (!PVE::JSONSchema::pve_verify_ip($ip, 1)) {
+ my $host = $ip;
+ eval { $ip = PVE::Network::get_ip_from_hostname($host); };
+ if ($@) {
+ $error->("cannot use '$host': $@\n", 1) ;
+ return;
+ }
+ }
+
+ my $cidr = (Net::IP::ip_is_ipv6($ip)) ? "$ip/128" : "$ip/32";
+ my $configured_ips = PVE::Network::get_local_ip_from_cidr($cidr);
+
+ $error->("cannot use IP '$ip', it must be configured exactly once on local node!\n")
+ if (scalar(@$configured_ips) != 1);
+ };
+
+ $check_ip->($ring0_addr);
+ $check_ip->($ring1_addr);
+
+ warn "warning, ignore the following errors:\n$warnings" if $warnings;
+ die "detected the following error(s):\n$errors" if $errors;
+}
+
+my $backup_cfs_database = sub {
+ my ($dbfile) = @_;
+
+ mkdir $dbbackupdir;
+
+ print "backup old database\n";
+ my $ctime = time();
+ my $cmd = [
+ ['echo', '.dump'],
+ ['sqlite3', $dbfile],
+ ['gzip', '-', \ ">${dbbackupdir}/config-${ctime}.sql.gz"],
+ ];
+
+ PVE::Tools::run_command($cmd, 'errmsg' => "cannot backup old database\n");
+
+ # purge older backup
+ my $maxfiles = 10;
+ my @bklist = ();
+ foreach my $fn (<$dbbackupdir/config-*.sql.gz>) {
+ if ($fn =~ m!/config-(\d+)\.sql.gz$!) {
+ push @bklist, [$fn, $1];
+ }
+ }
+
+ @bklist = sort { $b->[1] <=> $a->[1] } @bklist;
+ while (scalar (@bklist) >= $maxfiles) {
+ my $d = pop @bklist;
+ print "delete old backup '$d->[0]'\n";
+ unlink $d->[0];
+ }
+};
+
+sub finish_join {
+ my ($nodename, $corosync_conf, $corosync_authkey) = @_;
+
+ mkdir "$localclusterdir";
+ PVE::Tools::file_set_contents($authfile, $corosync_authkey);
+ PVE::Tools::file_set_contents($localclusterconf, $corosync_conf);
+
+ print "stopping pve-cluster service\n";
+
+ system("umount $basedir -f >/dev/null 2>&1");
+ die "can't stop pve-cluster service\n" if system("systemctl stop pve-cluster") != 0;
+
+ $backup_cfs_database->($dbfile);
+ unlink $dbfile;
+
+ system("systemctl start pve-cluster") == 0 || die "starting pve-cluster failed\n";
+ system("systemctl start corosync");
+
+ # wait for quorum
+ my $printqmsg = 1;
+ while (!PVE::Cluster::check_cfs_quorum(1)) {
+ if ($printqmsg) {
+ print "waiting for quorum...";
+ STDOUT->flush();
+ $printqmsg = 0;
+ }
+ sleep(1);
+ }
+ print "OK\n" if !$printqmsg;
+
+ my $local_ip_address = PVE::Cluster::remote_node_ip($nodename);
+
+ print "generating node certificates\n";
+ PVE::Cluster::gen_pve_node_files($nodename, $local_ip_address);
+
+ print "merge known_hosts file\n";
+ PVE::Cluster::ssh_merge_known_hosts($nodename, $local_ip_address, 1);
+
+ print "restart services\n";
+ # restart pvedaemon and pveproxy (changed certs)
+ system("systemctl restart pvedaemon pveproxy");
+
+ print "successfully added node '$nodename' to cluster.\n";
+}
+
+
1;
--
2.11.0
More information about the pve-devel
mailing list