[pve-devel] [pve-container] LXC implement migration on replicated storage

[Fabian Grünbichler]

On Fri, Jun 09, 2017 at 08:19:51AM +0200, Wolfgang Link wrote:
> ---
>  src/PVE/LXC/Migrate.pm | 76 ++++++++++++++++++++++++++++++++++++++++++++++----
>  1 file changed, 70 insertions(+), 6 deletions(-)
> 
> diff --git a/src/PVE/LXC/Migrate.pm b/src/PVE/LXC/Migrate.pm
> index 2f0244b..cfca82c 100644
> --- a/src/PVE/LXC/Migrate.pm
> +++ b/src/PVE/LXC/Migrate.pm
> @@ -10,6 +10,10 @@ use PVE::INotify;
>  use PVE::Cluster;
>  use PVE::Storage;
>  use PVE::LXC;
> +use PVE::LXC::Config;
> +use PVE::ReplicationConfig;
> +use PVE::ReplicationState;
> +use JSON;
>  
>  use base qw(PVE::AbstractMigrate);
>  
> @@ -264,12 +268,42 @@ sub phase1 {
>  	die "can't migrate CT - check log\n";
>      }
>  
> +
> +    my $rep_cfg = PVE::ReplicationConfig->new();
> +    my $jobid;
> +    my $last_replica;
> +
> +    if ($jobid = PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $self->{node}) ) {
> +       my $stateobj = PVE::ReplicationState::read_state();
> +       my $state = PVE::ReplicationState::extract_job_state($stateobj, $rep_cfg->{ids}->{$jobid});
> +       $last_replica = PVE::ReplicationState::replication_snapshot_name($jobid, $state->{last_sync});
> +    }

IMHO this is a candidate to implement as helper somewhere?

> +
>      foreach my $volid (keys %$volhash) {
>  	my ($sid, $volname) = PVE::Storage::parse_volume_id($volid);
>  	push @{$self->{volumes}}, $volid;
> -	PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{ssh_info}, $sid);
> +	PVE::Storage::storage_migrate($self->{storecfg}, $volid, $self->{ssh_info}, $sid, undef, $last_replica);
> +    }
> +
> +
> +    # set new replica_target if we migrate to replica target.
> +
> +    if ($jobid) {
> +
> +       my $update_target = sub {
> +
> +           if ($rep_cfg->{ids}->{$jobid}->{target} eq $self->{node}) {
> +               $self->log('info', "change replica target to Node: $self->{opts}->{node}");
> +               $rep_cfg->{ids}->{$jobid}->{target} = $self->{opts}->{node};
> +               $rep_cfg->write();
> +           }
> +
> +       };
> +
> +       PVE::ReplicationConfig::lock($update_target);

this as well! but shouldn't this only happen after the migration was
successful? otherwise we need to roll it back in cleanup..

>      }
>  
> +
>      my $conffile = PVE::LXC::Config->config_file($vmid);
>      my $newconffile = PVE::LXC::Config->config_file($vmid, $self->{node});
>  
> @@ -312,13 +346,19 @@ sub phase3 {
>  
>      my $volids = $self->{volumes};
>  
> +    my $rep_cfg = PVE::ReplicationConfig->new();
> +    my $replica = PVE::ReplicationConfig::check_for_existing_jobs($rep_cfg, $vmid, 1);

not used?

> +    my $jobid =  PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $self->{opts}->{node});
>      # destroy local copies
>      foreach my $volid (@$volids) {
> -	eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
> -	if (my $err = $@) {
> -	    $self->log('err', "removing local copy of '$volid' failed - $err");
> -	    $self->{errors} = 1;
> -	    last if $err =~ /^interrupted by signal$/;
> +	# do not destroy if new target is local_host
> +	if (!$jobid) {
> +	    eval { PVE::Storage::vdisk_free($self->{storecfg}, $volid); };
> +	    if (my $err = $@) {
> +		$self->log('err', "removing local copy of '$volid' failed - $err");
> +		$self->{errors} = 1;
> +		last if $err =~ /^interrupted by signal$/;
> +	    }
>  	}
>      }
>  }
> @@ -341,6 +381,30 @@ sub final_cleanup {
>  	$self->cmd_logerr($cmd, errmsg => "failed to clear migrate lock");	
>      }
>  
> +    # transfer replication state to migration target node.
> +    my $rep_cfg = PVE::ReplicationConfig->new();
> +    my $new_target = $self->{opts}->{node};
> +    if (PVE::ReplicationConfig::get_jobid($rep_cfg, $vmid, $new_target)) {
> +	my $stateobj = PVE::ReplicationState::read_state();
> +
> +	my $old_target =  $self->{node};
> +
> +	my $vm_state = {};
> +	foreach my $key (keys %{$stateobj->{$vmid}}) {
> +	    if ($key =~ /^(.*\/)\Q$old_target\E$/) {
> +		$vm_state->{"$1$new_target"} = $stateobj->{$vmid}->{$key};
> +	    } else {
> +		$vm_state->{$key} = $stateobj->{$vmid}->{$key};
> +	    }
> +	}

this might warrant a helper as well?

also not sure about the order, shouldn't we transfer state first, then
flip the direction in the config, then delete the old source state?

> +
> +	# This have to be double quoted when it run it over ssh.
> +	my $state = PVE::Tools::shellquote(encode_json $vm_state);
> +
> +	my $cmd = [ @{$self->{rem_ssh}}, 'pvesr', 'set-state', $vmid, $state];
> +	$self->cmd_logerr($cmd, errmsg => "failed to activate replica");
> +    }
> +
>      # in restart mode, we start the container on the target node
>      # after migration
>      if ($self->{opts}->{restart} && $self->{was_running}) {
> -- 
> 2.11.0
> 
> 
> _______________________________________________
> pve-devel mailing list
> pve-devel at pve.proxmox.com
> https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel