[pve-devel] [PATCH storage] WIP: upload disk image

Timo Grodzinski t.grodzinski at profihost.ag
Mon Feb 1 12:51:01 CET 2016


Am 01.02.2016 um 09:45 schrieb Wolfgang Bumiller:
> I wonder if it makes more sense to just handle the
> ($content eq 'images') case in the existing upload call since this
> duplicates quite a lot of code and you might just have to adapt the
> $content if/else cases and add the '# convert image with qemu-img'
> portion conditionally to the worker for the image case? The 'vmid'
> option would have to be made optional and checked for the non-images
> case.

This is a good idea, but I think it's better to have different REST URLs
with different meanings for this.

I will check to put the __PACKAGE__->register_method call in a
parametrized sub like

sub register_upload_method {
  my ( $class, $name, $path, $validate_fn, $worker_fn ) = @_;
  __PACKAGE__->register_method(
    {
      name => $name,
      path => $path,
      ...
    }
  );
}

__PACKAGE__->register_upload_method(
'upload_image,'{storage}/upload_image', ... );

when the code will be more final.

> 
> 
> On Fri, Jan 29, 2016 at 06:42:18PM +0100, Timo Grodzinski wrote:
>> Signed-off-by: Timo Grodzinski <t.grodzinski at profihost.ag>
>> ---
>>  PVE/API2/Storage/Status.pm | 211 +++++++++++++++++++++++++++++++++++++++++++++
>>  PVE/Storage.pm             |   2 +-
>>  PVE/Storage/DirPlugin.pm   |   2 +-
>>  PVE/Storage/Plugin.pm      |   2 +-
>>  4 files changed, 214 insertions(+), 3 deletions(-)
>>
>> diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm
>> index 49bb58c..f887932 100644
>> --- a/PVE/API2/Storage/Status.pm
>> +++ b/PVE/API2/Storage/Status.pm
>> @@ -421,4 +421,215 @@ __PACKAGE__->register_method ({
>>  	return $upid;
>>     }});
>>      
>> +
>> +__PACKAGE__->register_method(
>> +    {
>> +        name        => 'upload_image',
>> +        path        => '{storage}/upload_image',
>> +        method      => 'POST',
>> +        description => "Upload templates and ISO images.",
>> +        permissions => {
>> +            check => [ 'perm', '/storage/{storage}', ['Datastore.AllocateTemplate'] ]
>> +            ,    # XXX [ 'Datastore.AllocateSpace' ]
>> +        },
>> +        protected  => 1,
>> +        parameters => {
>> +            additionalProperties => 0,
>> +            properties           => {
>> +                node => get_standard_option( 'pve-node' ),
>> +                vmid => get_standard_option( 'pve-vmid', { completion => \&PVE::QemuServer::complete_vmid } ),
>> +                storage => get_standard_option( 'pve-storage-id' ),    # temporary storage before qemu-img
>> +                format  => {
>> +                    description => "Target format for file storage.",
>> +                    type        => 'string',
>> +                    optional    => 1,
>> +                    enum        => [ 'raw', 'qcow2', 'vmdk' ],
>> +                },
>> +                content => {
>> +                    description => "Content type.",
>> +                    type        => 'string',
>> +                    format      => 'pve-storage-content',
>> +                },
>> +                filename => {
>> +                    description => "The name of the file to create.",
>> +                    type        => 'string',
>> +                },
>> +                tmpfilename => {
>> +                    description =>
>> +"The source file name. This parameter is usually set by the REST handler. You can only overwrite it when connecting to the trustet port on localhost.",
>> +                    type     => 'string',
>> +                    optional => 1,
>> +                },
>> +            },
>> +        },
>> +        returns => { type => "string" },
>> +        code    => sub {
>> +            my ( $param ) = @_;
>> +
>> +            my $rpcenv = PVE::RPCEnvironment::get();
>> +
>> +            my $user = $rpcenv->get_user();
>> +
>> +            my $cfg = cfs_read_file( "storage.cfg" );
>> +
>> +            # XXX use PVE::Tools::extract_param ?
>> +            my $node   = $param->{node};
>> +            my $vmid   = $param->{vmid};
>> +            my $format = $param->{format};
>> +
>> +            my $scfg = PVE::Storage::storage_check_enabled( $cfg, $param->{storage}, $node );
>> +
>> +            die "cant upload to storage type '$scfg->{type}'\n"
>> +              if !( $scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || $scfg->{type} eq 'glusterfs' );
>> +
>> +            my $content = $param->{content};
>> +
>> +            my $tmpfilename = $param->{tmpfilename};
>> +            die "missing temporary file name\n" if !$tmpfilename;
>> +
>> +            my $size = -s $tmpfilename;
>> +            die "temporary file '$tmpfilename' does not exists\n" if !defined( $size );
>> +
>> +            my $filename = $param->{filename};
>> +
>> +            chomp $filename;
>> +            $filename =~ s/^.*[\/\\]//;
>> +            $filename =~ s/[;:,=\s\x80-\xff]/_/g;
>> +
>> +            my $path;
>> +
>> +            if ( $content eq 'images' ) {    # XXX see also pve-storage/PVE/Storage/DirPlugin.pm and other
>> +
>> +                my @allowed_extensions = qw(raw qcow qcow2 cow vdi vmdk vpc cloop);
>> +                raise_param_exc(
>> +                    { filename => "extension must be one of " . join ', ', map "'$_'", @allowed_extensions } )
>> +                  if !grep { $filename =~ m![^/]+\.$_$! } @allowed_extensions;
>> +
>> +                # XXX use another location!
>> +                $path = PVE::Storage::get_vztmpl_dir( $cfg, $param->{storage} );
>> +            }
>> +            else {
>> +                raise_param_exc( { content => "upload content type '$content' not allowed" } );
>> +            }
>> +
>> +            die "storage '$param->{storage}' does not support '$content' content\n"
>> +              if !$scfg->{content}->{$content};
>> +
>> +            my $dest    = "$path/$filename";
>> +            my $dirname = dirname( $dest );
>> +
>> +            # we simply overwrite when destination when file already exists
>> +
>> +            # -- copy --
>> +
>> +            my $cmd;
>> +            if ( $node ne 'localhost' && $node ne PVE::INotify::nodename() ) {
>> +                my $remip = PVE::Cluster::remote_node_ip( $node );
>> +
>> +                my @ssh_options = ( '-o', 'BatchMode=yes' );
>> +
>> +                my @remcmd = ( '/usr/bin/ssh', @ssh_options, $remip, '--' );
>> +
>> +                eval {
>> +                    # activate remote storage
>> +                    PVE::Tools::run_command( [ @remcmd, '/usr/sbin/pvesm', 'status', '--storage', $param->{storage} ] );
>> +                };
>> +                die "can't activate storage '$param->{storage}' on node '$node'\n" if $@;
>> +
>> +                PVE::Tools::run_command( [ @remcmd, '/bin/mkdir', '-p', '--', PVE::Tools::shell_quote( $dirname ) ],
>> +                    errmsg => "mkdir failed" );
>> +
>> +                $cmd =
>> +                  [ '/usr/bin/scp', @ssh_options, '--', $tmpfilename, "[$remip]:" . PVE::Tools::shell_quote( $dest ) ];
>> +            }
>> +            else {
>> +                PVE::Storage::activate_storage( $cfg, $param->{storage} );
>> +                File::Path::make_path( $dirname );
>> +                $cmd = [ 'cp', '--', $tmpfilename, $dest ];
>> +            }
>> +
>> +            # -- import --
>> +
>> +            die "vm $vmid is running\n" if PVE::QemuServer::check_running( $vmid );    # XXX
>> +
>> +            my $conf     = PVE::QemuServer::load_config( $vmid );
>> +            my $storecfg = PVE::Storage::config();
>> +            my $full     = 1;
>> +
>> +            my ( $drivename, $drive );
>> +            for my $opt ( keys %{$conf} ) {
>> +                if ( PVE::QemuServer::valid_drivename( $opt ) ) {
>> +                    $drivename = $opt;
>> +                    $drive = PVE::QemuServer::parse_drive( $opt, $conf->{$opt} );
>> +                    die "unable to parse drive options for '$opt'\n" if !$drive;
>> +                    next if PVE::QemuServer::drive_is_cdrom( $drive );
>> +                    die "Full clone feature is not available"
>> +                      if $full && !PVE::Storage::volume_has_feature( $storecfg, 'copy', $drive->{file}, undef, undef );
>> +                    last;
>> +                }
>> +            }
>> +
>> +            my $storage = PVE::Storage::parse_volume_id( $drive->{file} );
>> +
>> +            my $worker = sub {
>> +                my $upid = shift;
>> +
>> +                # copy file to storage
>> +                print "starting file import from: $tmpfilename\n";
>> +                print "target node: $node\n";
>> +                print "target file: $dest\n";
>> +                print "file size is: $size\n";
>> +                print "command: " . join( ' ', @$cmd ) . "\n";
>> +
>> +                eval { PVE::Tools::run_command( $cmd, errmsg => 'import failed' ); };
>> +                if ( my $err = $@ ) {
>> +                    unlink $dest;
>> +                    die $err;
>> +                }
>> +                print "finished file import successfully\n";
>> +
>> +                # convert image with qemu-img
>> +                print "\n";
>> +                print "starting import of image: $dest\n";
>> +
>> +                # my $source_file = "$param->{storage}:$dest";
>> +                my $source = PVE::Storage::path_to_volume_id( $storecfg, $dest );
>> +
>> +                # XXX remove
>> +                use Data::Dumper;
>> +                warn 'PVE::API2::Storage::Status: ' . Dumper(
>> +                    {
>> +                        filename => $filename,
>> +                        dest     => $dest,
>> +                        dirname  => dirname( $dest ),
>> +                        node     => $node,
>> +                        vmid     => $vmid,
>> +                        conf     => $conf,
>> +                        storecfg => $storecfg,
>> +                        drive    => $drive,
>> +                        storage  => $storage,
>> +                        #source_file => $source_file,
>> +                        source => $source,
>> +                    }
>> +                );
>> +
>> +                my $newvollist = [];
>> +                my $newdrive = PVE::QemuServer::import_disk( $source, $storecfg, $vmid, $drive, $newvollist );
>> +
>> +                print Dumper $newdrive;
>> +
>> +            };
>> +
>> +            my $upid = $rpcenv->fork_worker( 'imgcopy', undef, $user, $worker ); # XXX other name than imgcopy
>> +
>> +            # apache removes the temporary file on return, so we need
>> +            # to wait here to make sure the worker process starts and
>> +            # opens the file before it gets removed.
>> +            sleep( 1 );
>> +
>> +            return $upid;
>> +          }
>> +    }
>> +);
>> +
>>  1;
>> diff --git a/PVE/Storage.pm b/PVE/Storage.pm
>> index 140f8ae..e112d4e 100755
>> --- a/PVE/Storage.pm
>> +++ b/PVE/Storage.pm
>> @@ -372,7 +372,7 @@ sub path_to_volume_id {
>>  	} elsif ($path =~ m!^$isodir/([^/]+\.[Ii][Ss][Oo])$!) {
>>  	    my $name = $1;
>>  	    return ('iso', "$sid:iso/$name");
>> -	} elsif ($path =~ m!^$tmpldir/([^/]+\.tar\.gz)$!) {
>> +	} elsif ($path =~ m!^$tmpldir/([^/]+(:?\.tar\.gz|\.vmdk))$!) {
>>  	    my $name = $1;
>>  	    return ('vztmpl', "$sid:vztmpl/$name");
>>  	} elsif ($path =~ m!^$privatedir/(\d+)$!) {
>> diff --git a/PVE/Storage/DirPlugin.pm b/PVE/Storage/DirPlugin.pm
>> index bc3c61f..87e9cd7 100644
>> --- a/PVE/Storage/DirPlugin.pm
>> +++ b/PVE/Storage/DirPlugin.pm
>> @@ -16,7 +16,7 @@ sub type {
>>  
>>  sub plugindata {
>>      return {
>> -	content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1, none => 1 },
>> +	content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1, none => 1 }, # XXX maybe add new content type?
>>  		     { images => 1,  rootdir => 1 }],
>>  	format => [ { raw => 1, qcow2 => 1, vmdk => 1, subvol => 1 } , 'raw' ],
>>      };
>> diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
>> index 6aa71e0..a8bb385 100644
>> --- a/PVE/Storage/Plugin.pm
>> +++ b/PVE/Storage/Plugin.pm
>> @@ -374,7 +374,7 @@ sub parse_volname {
>>  	return ('images', $name, $vmid, undef, undef, $isBase, $format);
>>      } elsif ($volname =~ m!^iso/([^/]+\.[Ii][Ss][Oo])$!) {
>>  	return ('iso', $1);
>> -    } elsif ($volname =~ m!^vztmpl/([^/]+\.tar\.[gx]z)$!) {
>> +    } elsif ($volname =~ m!^vztmpl/([^/]+(:?\.tar\.[gx]z|\.vmdk|\.qcow2|\.raw))$!) {
>>  	return ('vztmpl', $1);
>>      } elsif ($volname =~ m!^rootdir/(\d+)$!) {
>>  	return ('rootdir', $1, $1);
>> -- 
>> 2.1.4
>>
>> _______________________________________________
>> pve-devel mailing list
>> pve-devel at pve.proxmox.com
>> http://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
>>
> 


-- 
Mit freundlichen Grüßen
  Timo Grodzinski
Ihr Profihost Team

-------------------------------
Profihost AG
Expo Plaza 1
30539 Hannover
Deutschland

Tel.: +49 (511) 5151 8181     | Fax.: +49 (511) 5151 8282
URL: http://www.profihost.com | E-Mail: info at profihost.com

Sitz der Gesellschaft: Hannover, USt-IdNr. DE813460827
Registergericht: Amtsgericht Hannover, Register-Nr.: HRB 202350
Vorstand: Cristoph Bluhm, Sebastian Bluhm, Stefan Priebe
Aufsichtsrat: Prof. Dr. iur. Winfried Huck (Vorsitzender)



More information about the pve-devel mailing list