[pve-devel] [PATCH v6 pve-storage 1/6] First beta of FreeNAS storage plugin. Missing snapshot backup for LXC

mir at datanom.net mir at datanom.net
Mon Jun 19 16:44:22 CEST 2017


From: Michael Rasmussen <mir at datanom.net>

Signed-off-by: Michael Rasmussen <mir at datanom.net>
---
 PVE/Storage.pm               |    2 +
 PVE/Storage/FreeNASPlugin.pm | 1289 ++++++++++++++++++++++++++++++++++++++++++
 PVE/Storage/Makefile         |    2 +-
 PVE/Storage/Plugin.pm        |    2 +-
 4 files changed, 1293 insertions(+), 2 deletions(-)
 create mode 100644 PVE/Storage/FreeNASPlugin.pm

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index ee2295a..f33249f 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -32,6 +32,7 @@ use PVE::Storage::GlusterfsPlugin;
 use PVE::Storage::ZFSPoolPlugin;
 use PVE::Storage::ZFSPlugin;
 use PVE::Storage::DRBDPlugin;
+use PVE::Storage::FreeNASPlugin;
 
 # Storage API version. Icrement it on changes in storage API interface.
 use constant APIVER => 1;
@@ -49,6 +50,7 @@ PVE::Storage::GlusterfsPlugin->register();
 PVE::Storage::ZFSPoolPlugin->register();
 PVE::Storage::ZFSPlugin->register();
 PVE::Storage::DRBDPlugin->register();
+PVE::Storage::FreeNASPlugin->register();
 
 # load third-party plugins
 if ( -d '/usr/share/perl5/PVE/Storage/Custom' ) {
diff --git a/PVE/Storage/FreeNASPlugin.pm b/PVE/Storage/FreeNASPlugin.pm
new file mode 100644
index 0000000..771061e
--- /dev/null
+++ b/PVE/Storage/FreeNASPlugin.pm
@@ -0,0 +1,1289 @@
+package PVE::Storage::FreeNASPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use POSIX;
+use PVE::Tools qw(run_command);
+use PVE::Storage::Plugin;
+use PVE::RPCEnvironment;
+use PVE::QemuServer;
+use PVE::LXC;
+use LWP::UserAgent;
+use HTTP::Status;
+use JSON;
+use Data::Dumper;
+
+use base qw(PVE::Storage::Plugin);
+
+my $api = '/api/v1.0';
+my $timeout = 60; # seconds
+my $max_luns = 20; # maximum supported luns per target group in freenas is 25
+                   # but reserve 5 for temporary LUNs (snapshots with RAM and
+                   # snapshot backup)
+my $active_snaps = 4;
+my $limit = 10000; # limit for get requests
+my $version;
+my $fullversion;
+my $target_prefix = 'iqn.2005-10.org.freenas.ctl';
+
+sub freenas_request {
+    my ($scfg, $request, $section, $data, $valid_code) = @_;
+    my $ua = LWP::UserAgent->new;
+    $ua->agent("ProxmoxUA/0.1");
+    $ua->ssl_opts( verify_hostname => 0 );
+    $ua->timeout($timeout);
+    push @{ $ua->requests_redirectable }, 'POST';
+    push @{ $ua->requests_redirectable }, 'PUT';
+    push @{ $ua->requests_redirectable }, 'DELETE';
+    my $req;
+
+    my $url = "https://$scfg->{portal}$api/$section";
+
+    if ($request eq 'GET') {
+        $req = HTTP::Request->new(GET => $url);
+    } elsif ($request eq 'POST') {
+        $req = HTTP::Request->new(POST => $url);
+        $req->content($data);
+    } elsif ($request eq 'PUT') {
+        $req = HTTP::Request->new(PUT => $url);
+        $req->content($data);
+    } elsif ($request eq 'DELETE') {
+        $req = HTTP::Request->new(DELETE => $url);
+    } else {
+        die "$request: Unknown request";
+    }
+
+    $req->content_type('application/json');
+    $req->authorization_basic($scfg->{username}, $scfg->{password});
+    my $res = $ua->request($req);
+
+	#print Dumper($res);	
+    return $res->code unless $res->is_success;
+
+    return $res->content;
+}
+
+sub freenas_get_version {
+    my ($scfg) = @_;
+	
+	my $response = freenas_request($scfg, 'GET', "system/version");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $info = decode_json($response);
+	$fullversion = $info->{fullversion};
+	if ($fullversion =~ /^\w+-(\d+)\.(\d*)\.(\d*)/) {
+		my $minor = $2;
+		my $micro = $3;
+
+		if ($minor) {
+			$minor = "0$minor" unless $minor > 9;
+		} else {
+			$minor = '00';
+		}
+		
+		if ($micro) {
+			$micro = "0$micro" unless $micro > 9;
+		} else {
+			$micro = '00';
+		}
+			
+		$version = "$1$minor$micro";
+	} else {
+		$version = '90200';
+	}
+}
+
+sub freenas_list_zvol {
+    my ($scfg) = @_;
+
+	freenas_get_version($scfg) unless $version;
+	
+	my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}/zvols?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+    my $zvols = decode_json($response);
+	$response = freenas_request($scfg, 'GET', "storage/snapshot?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+    my $snapshots = decode_json($response);
+
+    my $list = ();
+    my $hide = {};
+    my $vmid;
+    my $parent;
+    foreach my $zvol (@$zvols) {
+    	next unless $zvol->{name} =~ /^(base|vm)-(\d+)-disk-\d+$/;
+    	$vmid = $2;
+    	$parent = undef;
+    	foreach my $snap (@$snapshots) {
+    		next unless $snap->{name} eq "__base__$vmid";
+    		$parent = $snap->{filesystem} =~ /^$scfg->{pool}\/(.+)$/ ? $1 : undef;
+    	}
+        $list->{$scfg->{pool}}->{$zvol->{name}} = {
+            name => $zvol->{name},
+            size => $zvol->{volsize},
+            parent => $parent,
+            vmid => $vmid,
+            format => 'raw',
+        };
+        if ($zvol->{name} =~ /^base-(.*)/) {
+        	$hide->{"vm-$1"} = 1;
+        }
+    }
+
+    delete @{$list->{$scfg->{pool}}}{keys %$hide};
+	
+    return $list;
+}
+
+sub freenas_no_more_extents {
+	my ($scfg, $target) = @_;
+	
+	my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $extents = decode_json($response);
+	foreach my $extent (@$extents) {
+		return 0 if $extent->{iscsi_target} == $target;
+	}
+	
+	return 1;
+}
+	
+sub freenas_get_target {
+	my ($scfg, $vmid) = @_;
+	my $target = undef;
+	
+	my $response = freenas_request($scfg, 'GET', "services/iscsi/target?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $targets = decode_json($response);
+	foreach my $t (@$targets) {
+		if ($t->{iscsi_target_name} eq "vm-$vmid") {
+			$target = $t->{id};
+			last;
+		}
+	}
+	
+	return $target;
+}
+
+sub freenas_create_target {
+	my ($scfg, $vmid, $valid_code) = @_;
+	my $data;
+	
+	freenas_get_version($scfg) unless $version;
+
+	if ($version < 110000) {
+		$data = {
+			iscsi_target_alias => "vm-$vmid",
+			iscsi_target_name => "vm-$vmid",
+		};
+	} elsif ($version < 110100) {
+		$data = {
+			iscsi_target_alias => "vm-$vmid",
+			iscsi_target_name => "vm-$vmid",
+		};
+	} else {
+		die "FreeNAS-$version: Unsupported!";
+	}
+	my $response = freenas_request($scfg, 'POST', "services/iscsi/target", encode_json($data), $valid_code);
+	if ($response =~ /^\d+$/) {
+		return freenas_get_target($scfg, $vmid) if $valid_code && $response == $valid_code;
+		die HTTP::Status::status_message($response);
+	}
+	my $target = decode_json($response);
+	
+	die "Creating target for 'vm-$vmid' failed" unless $target->{id};
+	
+	return $target->{id};
+}
+
+sub freenas_delete_target {
+	my ($scfg, $target) = @_;
+
+	my $response = freenas_request($scfg, 'DELETE', "services/iscsi/target/$target");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_name {
+	my ($scfg, $volname) = @_;
+	my $name = undef;
+	
+	if ($volname =~ /^(vm|base)-(\d+)-/) {
+		$name = "vm-$2";
+		return "$target_prefix\:$name";
+	}
+
+	return undef;
+}
+
+sub freenas_create_target_group {
+	my ($scfg, $target, $valid_code) = @_;
+	my $data;
+	
+	freenas_get_version($scfg) unless $version;
+
+	# Trying to create a target group which already exists will cause and internal
+	# server error so if creating an existing target group should be allowed (return
+	# existing target group number we must search prior to create
+	if ($valid_code && $valid_code == 409) {
+		my $tg = freenas_get_target_group($scfg, $target);
+		return $tg if $tg;
+	}
+	
+	if ($version < 110000) {
+		$data = {
+			iscsi_target => $target,
+			iscsi_target_authgroup => $scfg->{auth_group} ? $scfg->{auth_group} : undef,
+			iscsi_target_portalgroup => $scfg->{portal_group},
+			iscsi_target_initiatorgroup => $scfg->{initiator_group},
+			iscsi_target_authtype => $scfg->{auth_type} ? $scfg->{auth_type} : 'None',
+			iscsi_target_initialdigest => "Auto",
+		};
+	} elsif ($version < 110100) {
+		$data = {
+			iscsi_target => $target,
+			iscsi_target_authgroup => $scfg->{auth_group} ? $scfg->{auth_group} : undef,
+			iscsi_target_portalgroup => $scfg->{portal_group},
+			iscsi_target_initiatorgroup => $scfg->{initiator_group},
+			iscsi_target_authtype => $scfg->{auth_type} ? $scfg->{auth_type} : 'None',
+			iscsi_target_initialdigest => "Auto",
+		};
+	} else {
+		die "FreeNAS-$version: Unsupported!";
+	}
+
+	my $response = freenas_request($scfg, 'POST', "services/iscsi/targetgroup", encode_json($data), $valid_code);
+	if ($response =~ /^\d+$/) {
+		if ($valid_code != 409) {
+			return freenas_get_target_group($scfg, $target) if $valid_code && $response == $valid_code;
+		}
+		die HTTP::Status::status_message($response);
+	}
+	my $tg = decode_json($response);
+	
+	die "Creating target group for target '$target' failed" unless $tg->{id};
+	
+	return $tg->{id};
+}
+
+sub freenas_delete_target_group {
+	my ($scfg, $tg) = @_;
+
+	my $response = freenas_request($scfg, 'DELETE', "services/iscsi/targetgroup/$tg");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_group {
+	my ($scfg, $target) = @_;
+	my $targetgroup = undef;
+	
+	my $response = freenas_request($scfg, 'GET', "services/iscsi/targetgroup?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $targetgroups = decode_json($response);
+
+	foreach my $tgroup (@$targetgroups) {
+		if ($tgroup->{iscsi_target} == $target && 
+			$tgroup->{iscsi_target_portalgroup} == $scfg->{portal_group} &&
+			$tgroup->{iscsi_target_initiatorgroup} == $scfg->{initiator_group}) {
+			$targetgroup = $tgroup->{id};
+			last;
+		}
+	}
+	
+	return $targetgroup;
+}
+
+sub freenas_create_extent {
+	my ($scfg, $zvol) = @_;
+	my $data;
+	
+	freenas_get_version($scfg) unless $version;
+	
+	if ($version < 110000) {
+		$data = {
+			iscsi_target_extent_type => 'Disk',
+			iscsi_target_extent_name => $zvol,
+			iscsi_target_extent_disk => "zvol/$scfg->{pool}/$zvol",
+		};
+	} elsif ($version < 110100) {
+		$data = {
+			iscsi_target_extent_type => 'Disk',
+			iscsi_target_extent_name => $zvol,
+			iscsi_target_extent_disk => "zvol/$scfg->{pool}/$zvol",
+		};
+	} else {
+		die "FreeNAS-$version: Unsupported!";
+	}
+
+	my $response = freenas_request($scfg, 'POST', "services/iscsi/extent", encode_json($data));
+	die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+	my $extent = decode_json($response);
+	
+	die "Creating LUN for volume '$zvol' failed" unless $extent->{id};
+	
+	return $extent->{id};
+}
+
+sub freenas_delete_extent {
+	my ($scfg, $extent) = @_;
+
+	my $response = freenas_request($scfg, 'DELETE', "services/iscsi/extent/$extent");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_extent {
+	my ($scfg, $volname) = @_;
+	my $extent = undef;
+	
+	my $response = freenas_request($scfg, 'GET', "services/iscsi/extent?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $extents = decode_json($response);
+	foreach my $ext (@$extents) {
+		if ($ext->{iscsi_target_extent_path} =~ /$scfg->{pool}\/$volname$/) {
+			$extent = $ext->{id};
+			last;
+		}
+	}
+	
+	return $extent;
+}
+
+sub freenas_create_target_to_exent {
+	my ($scfg, $target, $extent, $lunid) = @_;
+	my $data;
+	
+	freenas_get_version($scfg) unless $version;
+	
+	if ($version < 110000) {
+		$data = {
+			iscsi_target => $target,
+			iscsi_extent => $extent,
+			iscsi_lunid => $lunid,
+		};
+	} elsif ($version < 110100) {
+		$data = {
+			iscsi_target => $target,
+			iscsi_extent => $extent,
+			iscsi_lunid => $lunid,
+		};
+	} else {
+		die "FreeNAS-$version: Unsupported!";
+	}
+
+	my $response = freenas_request($scfg, 'POST', "services/iscsi/targettoextent", encode_json($data));
+	die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+	my $tg2extent = decode_json($response);
+	
+	die "Creating view for LUN '$extent' failed" unless $tg2extent->{id};
+	
+	return $tg2extent->{id};
+}
+
+sub freenas_delete_target_to_exent {
+	my ($scfg, $tg2exent) = @_;
+
+	my $response = freenas_request($scfg, 'DELETE', "services/iscsi/targettoextent/$tg2exent");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub freenas_get_target_to_exent {
+	my ($scfg, $extent, $target) = @_;
+	my $t2extent = undef;
+	
+	my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $t2extents = decode_json($response);
+	foreach my $t2ext (@$t2extents) {
+		if ($t2ext->{iscsi_target} == $target && $t2ext->{iscsi_extent} == $extent) {
+			$t2extent = $t2ext->{id};
+			last;
+		}
+	}
+	
+	return $t2extent;
+}
+
+sub freenas_find_free_diskname {
+    my ($storeid, $scfg, $vmid, $format) = @_;
+
+    my $name = undef;
+    my $volumes = freenas_list_zvol($scfg);
+
+    my $disk_ids = {};
+    my $dat = $volumes->{$scfg->{pool}};
+
+   	foreach my $image (keys %$dat) {
+       	my $volname = $dat->{$image}->{name};
+       	if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
+       	    $disk_ids->{$2} = 1;
+       	}
+   	}
+
+   	for (my $i = 1; $i < $max_luns + 1; $i++) {
+   	    if (!$disk_ids->{$i}) {
+   	        return "vm-$vmid-disk-$i";
+   	    }
+   	}
+    	
+   	die "Maximum number of LUNs($max_luns) for this VM $vmid in storage '$storeid' is reached.";
+}
+
+sub freenas_get_lun_number {
+    my ($scfg, $volname, $snap) = @_;
+    my $lunid = undef;
+    
+    if ($volname =~ /^(vm|base)-\d+-disk-(\d+)$/ && ! defined $snap) {
+    	$lunid = $2 - 1;
+    } elsif ($volname =~ /^vm-(\d+)-state/) {
+    	# Find id for temporary LUN
+    	if ($snap) {
+    		# TODO
+    		# Required to be able to exposed read-only LUNs for snapshot backup CT
+    	} else {
+    		my $target = freenas_get_target($scfg, $1);
+    		my $id = $max_luns;
+			my $response = freenas_request($scfg, 'GET', "services/iscsi/targettoextent?limit=$limit");
+			die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+			my $t2extents = decode_json($response);
+
+			foreach my $t2extent (@$t2extents) {
+				next unless $t2extent->{iscsi_target} == $target &&
+							$t2extent->{iscsi_lunid} + 1 > $max_luns &&
+							$t2extent->{iscsi_lunid} < $max_luns + $active_snaps;
+				my $eid = freenas_get_extent($scfg, $volname);
+				if ($eid) {
+					$response = freenas_request($scfg, 'GET', "services/iscsi/extent/$eid");
+					die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+					my $extent = decode_json($response);
+					# Request to get lunid for an existing lun
+					last if $t2extent->{iscsi_extent} eq $eid;
+				}
+				$id++;
+			}
+			die "Max snapshots (4) is reached" unless ($id - $max_luns) < $active_snaps;
+			$lunid = $id;
+    	}
+    }
+    
+    return $lunid;
+}
+
+sub freenas_create_lun {
+	my ($scfg, $vmid, $zvol) = @_;
+	my ($target, $tg, $extent, $tg2exent) = (undef, undef, undef, undef);
+	
+	eval {
+		$target = freenas_create_target($scfg, $vmid, 409);
+		die "create_lun-> Could not create target for VM '$vmid'" unless $target;
+		$tg = freenas_create_target_group($scfg, $target, 409);
+		die "create_lun-> Could not create target group for VM '$vmid'" unless $tg;
+		$extent = freenas_create_extent($scfg, $zvol);
+		die "create_lun-> Could not create extent for VM '$vmid'" unless $extent;
+		my $lunid = freenas_get_lun_number($scfg, $zvol);
+		die "create_lun-> $zvol: Bad name format for VM '$vmid'" unless defined $lunid;
+		$tg2exent = freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+		die "create_lun-> Could not create target to extend for VM '$vmid'" unless defined $tg2exent;
+	};
+	if ($@) {
+		my $err = $@;
+		if ($tg2exent) {
+			freenas_delete_target_to_exent($scfg, $tg2exent);
+		}
+		if ($extent) {
+			freenas_delete_extent($scfg, $extent);
+		}
+		if ($target && freenas_no_more_extents($scfg, $target)) {
+			if ($tg) {
+				freenas_delete_target_group($scfg, $tg);
+			}
+			freenas_delete_target($scfg, $target);
+		}
+		die $err;
+	}
+}
+
+sub freenas_create_zvol {
+	my ($scfg, $volname, $size) = @_;
+	
+	my $data = {
+		name => $volname,
+		volsize => $size,
+	};
+	my $response = freenas_request($scfg, 'POST', "storage/volume/$scfg->{pool}/zvols", encode_json($data));
+	die HTTP::Status::status_message($response) if ($response =~ /^\d+$/);
+	my $zvol = decode_json($response);
+
+	die "$volname: Failed creating volume" unless $zvol && $zvol->{name};
+	
+	return $zvol->{name};
+}
+
+sub freenas_delete_zvol {
+	my ($scfg, $volname) = @_;
+
+	my $response = freenas_request($scfg, 'DELETE', "storage/volume/$scfg->{pool}/zvols/$volname");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub os_request {
+	my ($cmd, $noerr, $timeout) = @_;
+
+	$timeout = PVE::RPCEnvironment::is_worker() ? 60*60 : 5 if !$timeout;
+	$noerr = 0 if !$noerr;
+
+    my $text = '';
+
+    my $output = sub {
+        my $line = shift;
+        $text .= "$line\n";
+    };
+
+   	my $exit_code = run_command($cmd, noerr => $noerr, errfunc => $output, outfunc => $output, timeout => $timeout);
+
+    return wantarray ? ($exit_code, $text) : $exit_code;
+}
+
+sub bail_out {
+	my ($class, $storeid, $scfg, $volname, $err) = @_;
+	
+	$class->free_image($storeid, $scfg, $volname);
+	die $err;
+}
+
+sub disk_by_path {
+	my ($scfg, $volname) = @_;
+
+	my $target = freenas_get_target_name($scfg, $volname);
+	my $lun = freenas_get_lun_number($scfg, $volname);
+	my $path = "/dev/disk/by-path/ip-$scfg->{portal}\:3260-iscsi-$target-lun-$lun";
+
+	return $path;
+}
+
+sub build_lun_list {
+	my ($scfg, $sid, $lun) = @_;
+
+	my $luns = {};
+	my $text = '';
+	my $exit = 0;
+
+	eval {
+		($exit, $text) = os_request("iscsiadm -m session -r $sid -P3", 1, 60);
+	};
+	if ($@) {
+		# An exist code of 22 means no active session otherwise an error
+		if ($exit != 22) {
+			die "$@";
+		}
+	}
+	if ($text =~ /.*Host Number:\s*(\d+)\s+State:\s+running(.*)/s) {
+    	my $host = $1;
+    	my $found = 0;
+    	for (split /^/, $2) {
+        	if ($_ =~ /Channel\s+(\d+)\s+Id\s+(\d+)\s+Lun:\s+(\d+)/) {
+        		if (defined $lun && $lun == $3) {
+        			$luns = {};
+        			$found = 1;
+        		}
+        		$luns->{$3} = "$host:".int($1).":$2:$3";
+        		last if $found;
+        	}
+    	}
+	}
+
+	return $luns;
+}
+
+sub get_sid {
+	my ($scfg, $volname) = @_;
+	my $sid = -1;
+	my $text = '';
+	my $exit = 0;
+	
+	my $target = freenas_get_target_name($scfg, $volname);
+
+	eval {
+		($exit, $text) = os_request("iscsiadm -m node -T $target -p $scfg->{portal} -s", 1, 60);
+	};
+	if ($@) {
+		# An exist code of 21 or 22 means no active session otherwise an error
+		if ($exit != 21 || $exit != 22) {
+			die "$@";
+		}
+	}
+	if ($text =~ /.*\[sid\:\s*(\d+),\s*.*/) {
+		$sid = $1;
+	}
+
+	return $sid;
+}
+
+sub create_session {
+	my ($scfg, $volname) = @_;
+	my $sid = -1;
+	my $exit = undef;
+	
+	my $target = freenas_get_target_name($scfg, $volname);
+
+	eval {
+		$exit = os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 1, 60);
+		if ($exit == 21) {
+			eval {
+				os_request("iscsiadm -m discovery -t sendtargets -p $scfg->{portal}", 0, 60);
+				os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 0, 60);
+			};
+		}
+	};
+	if ($@) {
+		if ($exit == 21) {
+			eval {
+				os_request("iscsiadm -m discovery -t sendtargets -p $scfg->{portal}", 0, 60);
+				os_request("iscsiadm -m node -T $target -p $scfg->{portal} --login", 0, 60);
+			};
+		} else {
+			die $@;
+		}
+	}
+	eval {
+		$sid = get_sid($scfg, $volname);
+	};
+	die "$@" if $@;
+	die "Could not create session" if $sid < 0;
+	
+	return $sid;
+}
+
+sub delete_session {
+	my ($scfg, $sid) = @_;
+	
+	eval {
+		os_request("iscsiadm -m session -r $sid --logout", 0, 60);
+	};
+}
+
+sub remove_local_lun {
+	my ($id) = @_;
+	
+	os_request("echo 1 > /sys/bus/scsi/devices/$id/delete", 0, 60);
+}
+
+sub deactivate_luns {
+	# $luns contains a hash of luns to keep
+	my ($scfg, $volname, $luns) = @_;
+
+	$luns = {} if !$luns;
+	my $sid;
+	my $list = {};
+
+	eval {		
+		$sid = get_sid($scfg, $volname);
+	};
+	die "$@" if $@;
+
+	eval {
+		$list = build_lun_list($scfg, $sid);
+
+		foreach my $key (keys %$list) {
+			next if exists($luns->{$key});
+			remove_local_lun($list->{$key});
+		}
+	};
+	die "$@" if $@;
+}
+
+sub get_active_luns {
+	my ($class, $storeid, $scfg, $volname) = @_;
+
+	my $sid = 0;
+	my $luns = {};
+
+	eval {
+		$sid = get_sid($scfg, $volname);
+	};
+	die "$@" if $@;
+	if ($sid < 0) {
+		# We have no active sessions so make one
+		eval {
+			$sid = create_session($scfg, $volname);
+		};
+		die "$@" if $@;
+		# Since no session existed prior to this call deactivate all LUN's found
+		deactivate_luns($scfg, $volname);
+	} else {
+		eval {
+			$luns = build_lun_list($scfg, $sid);
+		};
+		die "$@" if $@;
+	}
+
+	return $luns;
+}
+
+sub rescan_session {
+	my ($class, $storeid, $scfg, $volname, $exclude_lun) = @_;
+
+	eval {
+		my $active_luns = get_active_luns($class, $storeid, $scfg, $volname);
+		delete $active_luns->{$exclude_lun} if defined $exclude_lun;
+		my $sid = get_sid($scfg, $volname);
+		die "Missing session" if $sid < 0;
+		os_request("iscsiadm -m session -r $sid -R", 0, 60);
+		deactivate_luns($scfg, $volname, $active_luns);
+		delete_session($scfg, $sid) if !%$active_luns;
+	};
+	die "$@" if $@;
+}
+
+sub freenas_get_latest_snapshot {
+    my ($class, $scfg, $volname) = @_;
+
+    my $vname = ($class->parse_volname($volname))[1];
+
+    # abort rollback if snapshot is not the latest
+	my $response = freenas_request($scfg, 'GET', "storage/snapshot");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $snapshots = decode_json($response);
+	
+    my $recentsnap;
+    foreach my $snapshot (@$snapshots) {
+    	next unless $snapshot->{filesystem} =~ /$scfg->{pool}\/$vname/ && $snapshot->{mostrecent};
+        $recentsnap = $snapshot->{name};
+        last;
+    }
+
+    return $recentsnap;
+}
+    
+# Configuration
+
+sub type {
+    return 'freenas';
+}
+
+sub plugindata {
+    return {
+		content => [ {images => 1, rootdir => 1}, {images => 1 , rootdir => 1} ],
+		format => [ { raw => 1 } , 'raw' ],
+    };
+}
+
+sub properties {
+    return {
+        password => {
+        	description => "password",
+        	type => "string",
+        },
+        portal_group => {
+        	description => "Portal Group ID",
+        	type => "integer",
+        },
+        initiator_group => {
+        	description => "Initiator Group ID",
+        	type => "integer",
+        },
+    };
+}
+
+sub options {
+    return {
+		portal => { fixed => 1 },
+		pool => { fixed => 1 },
+		portal_group => { fixed => 1 },
+		initiator_group => { fixed => 1 },
+		blocksize => { optional => 1 },
+    	username => { optional => 1 },
+    	password => { optional => 1 },
+#		sparse => { optional => 1 }, not available in 9.2.x. Appear in 11.x
+#                                    in 9.2.x all zvols are created sparse!
+    	nodes => { optional => 1 },
+    	disable => { optional => 1 },
+		content => { optional => 1 },
+    };
+}
+
+# Storage implementation
+
+sub volume_size_info {
+    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+	my (undef, $vname) = $class->parse_volname($volname);
+
+	my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}/zvols/$vname");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $zvol = decode_json($response);
+	
+	return $zvol->{volsize} if $zvol && $zvol->{volsize};
+	
+    die "Could not get zfs volume size\n";
+}
+
+sub parse_volname {
+    my ($class, $volname) = @_;
+
+    if ($volname =~ m/^(((base)-(\d+)-\S+)\/)?((base|vm)-(\d+)-\S+)$/) {
+		my $format = 'raw';
+		my $isBase = ($6 eq 'base');
+		return ('images', $5, $7, $2, $4, $isBase, $format);
+    }
+
+    die "unable to parse freenas volume name '$volname'\n";
+}
+
+sub status {
+    my ($class, $storeid, $scfg, $cache) = @_;
+    my $total = 0;
+    my $free = 0;
+    my $used = 0;
+    my $active = 0;
+    
+    eval {
+		my $response = freenas_request($scfg, 'GET', "storage/volume/$scfg->{pool}");
+		die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+		my $vol = decode_json($response);
+        my $children = $vol->{children};
+        if (@$children) {
+            $used = $children->[0]{used};
+            $total = $children->[0]{avail};
+        } else {
+            $used = $vol->{used};
+            $total = $vol->{avail};
+        }
+        $free = $total - $used;
+        $active = 1;
+    };
+    warn $@ if $@;
+
+    return ($total, $free, $used, $active);
+}
+
+sub list_images {
+    my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+    $cache->{freenas} = freenas_list_zvol($scfg) if !$cache->{freenas};
+    my $zfspool = $scfg->{pool};
+    my $res = [];
+
+    if (my $dat = $cache->{freenas}->{$zfspool}) {
+
+		foreach my $image (keys %$dat) {
+
+			my $info = $dat->{$image};
+			my $volname = $info->{name};
+			my $parent = $info->{parent};
+			my $owner = $info->{vmid};
+			
+			if ($parent) {
+				$info->{volid} = "$storeid:$parent/$volname";
+			} else {
+				$info->{volid} = "$storeid:$volname";
+			}
+			
+			if ($vollist) {
+				my $found = grep { $_ eq $info->{volid} } @$vollist;
+				next if !$found;
+			} else {
+				next if defined ($vmid) && ($owner ne $vmid);
+			}
+			push @$res, $info;
+		}
+    }
+
+    return $res;
+}
+
+sub path {
+    my ($class, $scfg, $volname, $storeid, $snapname) = @_;
+
+    die "direct access to snapshots not implemented"
+	if defined($snapname);
+
+    my ($vtype, $vname, $vmid) = $class->parse_volname($volname);
+
+	my $luns = get_active_luns($class, $storeid, $scfg, $vname);
+	my $path = disk_by_path($scfg, $vname);
+	
+    return ($path, $vmid, $vtype);
+}
+
+sub create_base {
+    my ($class, $storeid, $scfg, $volname) = @_;
+    my ($lun, $target);
+    my $snap = '__base__';
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+        $class->parse_volname($volname);
+
+    die "create_base not possible with base image\n" if $isBase;
+
+    my $newname = $name;
+    $newname =~ s/^vm-/base-/;
+
+	$target = freenas_get_target($scfg, $vmid);
+	die "create_base-> missing target" unless $target;
+	my $extent = freenas_get_extent($scfg, $name);
+	die "create_base-> missing extent" unless $extent;
+	my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+	die "create_base-> missing target to extent" unless $tg2exent;
+	$lun = freenas_get_lun_number($scfg, $name);
+	die "create_base-> missing LUN" unless defined $lun;
+	freenas_delete_target_to_exent($scfg, $tg2exent);
+	freenas_delete_extent($scfg, $extent);
+	my $sid = get_sid($scfg, $name);
+	if ($sid >= 0) {
+		my $lid = build_lun_list($scfg, $sid, $lun);
+		if ($lid && $lid->{$lun}) {
+			remove_local_lun($lid->{$lun});
+		}
+	}
+
+	eval {
+		# FreeNAS API does not support renaming a zvol so create a snapshot
+		# and make a clone of the snapshot instead
+		$class->volume_snapshot($scfg, $storeid, $name, $snap);
+	
+		my $data = {
+			name => "$scfg->{pool}/$newname"
+		};
+		my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$name\@$snap/clone/", encode_json($data));
+		die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+
+   		freenas_create_lun($scfg, $vmid, $newname);
+    };
+    if ($@) {
+		$extent = freenas_create_extent($scfg, $name);
+		die "create_base-> Could not create extent for VM '$vmid'" unless $extent;
+		$tg2exent = freenas_create_target_to_exent($scfg, $target, $extent, $lun);
+		die "create_base-> Could not create target to extend for VM '$vmid'" unless defined $tg2exent;
+    }
+
+    return $newname;
+}
+
+sub clone_image {
+    my ($class, $scfg, $storeid, $volname, $vmid, $snap) = @_;
+
+    $snap ||= "__base__$vmid";
+
+    my ($vtype, $basename, $basevmid, undef, undef, $isBase, $format) =
+        $class->parse_volname($volname);
+
+    die "clone_image only works on base images" if !$isBase;
+
+	my $run = PVE::QemuServer::check_running($basevmid);
+	if (!$run) {
+		$run = PVE::LXC::check_running($basevmid);
+	}
+	
+    my $name = freenas_find_free_diskname($storeid, $scfg, $vmid, $format);
+
+	$class->volume_snapshot($scfg, $storeid, $basename, $snap);
+	
+	my $data = {
+		name => "$scfg->{pool}/$name"
+	};
+	my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$basename\@$snap/clone/", encode_json($data));
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+
+	$name = "$basename/$name";
+    # get ZFS dataset name from PVE volname
+    my (undef, $clonedname) = $class->parse_volname($name);
+
+	freenas_create_lun($scfg, $vmid, $clonedname);
+	
+	my $res = $class->deactivate_volume($storeid, $scfg, $basename) unless $run;
+	warn "Could not deactivate volume '$basename'" unless $res;
+	
+    return $name;
+}
+
+sub alloc_image {
+    my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+    die "unsupported format '$fmt'" if $fmt ne 'raw';
+
+    die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
+    if $name && $name !~ m/^vm-$vmid-/;
+
+    my $volname = $name;
+
+    $volname = freenas_find_free_diskname($storeid, $scfg, $vmid, $fmt) if !$volname;
+    
+    # Size is in KB but Freenas wants in bytes
+    $size *= 1024;
+    my $zvol = freenas_create_zvol($scfg, $volname, $size);
+
+    eval {
+    	freenas_create_lun($scfg, $vmid, $zvol) if $zvol;
+    };
+    if ($@) {
+    	my $err = $@;
+    	eval {
+    		freenas_delete_zvol($scfg, $volname);
+    	};
+    	$err .= "\n$@" if $@;
+    	die $err;
+    }
+
+    return $volname;
+}
+
+sub free_image {
+    my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+
+    my ($vtype, $name, $vmid, $basename) = $class->parse_volname($volname);
+
+	eval {
+		my $target = freenas_get_target($scfg, $vmid);
+		die "free_image-> missing target" unless $target;
+		my $extent = freenas_get_extent($scfg, $name);
+		die "free_image-> missing extent" unless $extent;
+		my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+		die "free_image-> missing target to extent" unless $tg2exent;
+		my $target_group = freenas_get_target_group($scfg, $target);
+		die "free_image-> missing target group" unless $target_group;
+		my $lun = freenas_get_lun_number($scfg, $name);
+		die "free_image-> missing LUN" unless defined $lun;
+	
+		my $res = $class->deactivate_volume($storeid, $scfg, $volname);
+		warn "Could not deactivate volume '$volname'" unless $res;
+		freenas_delete_target_to_exent($scfg, $tg2exent);
+		freenas_delete_extent($scfg, $extent);
+		if ($target && freenas_no_more_extents($scfg, $target)) {
+			if ($target_group) {
+				freenas_delete_target_group($scfg, $target_group);
+			}
+			freenas_delete_target($scfg, $target);
+		}
+		freenas_delete_zvol($scfg, $name);
+		$class->volume_snapshot_delete($scfg, $storeid, $basename, "__base__$vmid")	if $basename;
+		if ($isBase) {
+			$basename = $name;
+			$basename =~ s/^base-/vm-/;
+			$class->volume_snapshot_delete($scfg, $storeid, $basename, '__base__')	if $basename;
+			freenas_delete_zvol($scfg, $basename);
+		}
+	};
+    if ($@) {
+    	my $err = $@;
+        freenas_create_lun($scfg, $vmid, $name) unless $isBase;
+        die $err;
+    }
+
+    return undef;
+}
+
+sub volume_resize {
+    my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+    die 'mode failure - unable to resize disk(s) on a running system due to FreeNAS bug.<br />
+		 See bug report: <a href="https://bugs.freenas.org/issues/24432" target="_blank">#24432</a><br />' 		if $running;
+    
+    my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+    my $data = {
+    	volsize => $size,
+    };
+	my $response = freenas_request($scfg, 'PUT', "storage/volume/$scfg->{pool}/zvols/$name", encode_json($data));
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	my $vol = decode_json($response);
+
+	my $sid = get_sid($scfg, $name);
+	if ($sid >= 0) {
+		eval {
+#### Required because of a bug in FreeNAS: https://bugs.freenas.org/issues/24432
+			my $targetname = freenas_get_target_name($scfg, $name);
+			die "volume_resize-> Missing target name" unless $targetname;
+			my $target = freenas_get_target($scfg, $vmid);
+			die "volume_resize-> Missing target" unless $target;
+			my $extent = freenas_get_extent($scfg, $name);
+			die "volume_resize-> Missing extent" unless $extent;
+			my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+			die "volume_resize-> Missing target to extent" unless $tg2exent;
+			my $lunid = freenas_get_lun_number($scfg, $name);
+			die "volume_resize-> Missing LUN" unless defined $lunid;
+			freenas_delete_target_to_exent($scfg, $tg2exent);
+			freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+#### Required because of a bug in FreeNAS: https://bugs.freenas.org/issues/24432
+			rescan_session($class, $storeid, $scfg, $name, $lunid);
+		};
+		die "$name: Resize with $size failed. ($@)\n" if $@;
+	}
+
+    return int($vol->{volsize}/1024);
+}
+
+sub volume_snapshot {
+    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+    my $vname = ($class->parse_volname($volname))[1];
+
+    my $data = {
+    	dataset => "$scfg->{pool}/$vname",
+    	name => $snap,
+    };
+	my $response = freenas_request($scfg, 'POST', "storage/snapshot", encode_json($data));
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub volume_snapshot_delete {
+    my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+    
+    my $vname = ($class->parse_volname($volname))[1];
+    
+    my $response = freenas_request($scfg, 'DELETE', "storage/snapshot/$scfg->{pool}/$vname\@$snap");
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+}
+
+sub volume_snapshot_rollback {
+    my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+	my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+	my $target = freenas_get_target($scfg, $vmid);
+	die "volume_resize-> Missing target" unless $target;
+	my $extent = freenas_get_extent($scfg, $name);
+	die "volume_resize-> Missing extent" unless $extent;
+	my $tg2exent = freenas_get_target_to_exent($scfg, $extent, $target);
+	die "volume_resize-> Missing target to extent" unless $tg2exent;
+	my $lunid = freenas_get_lun_number($scfg, $name);
+	die "volume_resize-> Missing LUN" unless defined $lunid;
+	freenas_delete_target_to_exent($scfg, $tg2exent);
+	freenas_delete_extent($scfg, $extent);
+	
+	my $data = {
+		force => bless( do{\(my $o = 0)}, 'JSON::XS::Boolean' ),
+	};
+    my $response = freenas_request($scfg, 'POST', "storage/snapshot/$scfg->{pool}/$name\@$snap/rollback/", encode_json($data));
+	die HTTP::Status::status_message($response) if $response =~ /^\d+$/;
+	
+	$extent = freenas_create_extent($scfg, $name);
+	freenas_create_target_to_exent($scfg, $target, $extent, $lunid);
+	rescan_session($class, $storeid, $scfg, $name, $lunid);
+}
+
+sub volume_rollback_is_possible {
+    my ($class, $scfg, $storeid, $volname, $snap) = @_; 
+    
+    my (undef, $name) = $class->parse_volname($volname);
+
+    my $recentsnap = $class->freenas_get_latest_snapshot($scfg, $name);
+    if ($snap ne $recentsnap) {
+		die "can't rollback, more recent snapshots exist";
+    }
+
+    return 1; 
+}
+
+sub volume_snapshot_list {
+    my ($class, $scfg, $storeid, $volname, $prefix) = @_;
+    # return an empty array if dataset does not exist.
+    die "Volume_snapshot_list is not implemented for FreeNAS.\n";
+}
+
+sub volume_has_feature {
+    my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+    my $features = {
+		snapshot => { current => 1, snap => 1},
+		clone => { base => 1},
+		template => { current => 1},
+		copy => { base => 1, current => 1},
+    };
+
+    my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+	$class->parse_volname($volname);
+
+    my $key = undef;
+
+    if ($snapname) {
+		$key = 'snap';
+    } else {
+		$key = $isBase ? 'base' : 'current';
+    }
+
+    return 1 if $features->{$feature}->{$key};
+
+    return undef;
+}
+
+sub activate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+    
+    return 1;
+}
+
+sub deactivate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    return 1;
+}
+
+# Procedure for activating a LUN:
+#
+# if session does not exist
+#	login to target
+#	deactivate all luns in session
+# get list of active luns
+# get lun number to activate
+# make list of our luns (active + new lun)
+# rescan session
+# deactivate all luns except our luns
+sub activate_volume {
+    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+
+	# TODO: FreeNAS supports exposing a ro LUN from a snapshot
+	# ERROR: Backup of VM xyz failed - unable to activate snapshot from remote zfs storage
+	# at /usr/share/perl5/PVE/Storage/ZFSPlugin.pm line 418.
+	# $vname\@$snapname
+	die "mode failure - unable to activate snapshot from remote zfs storage" if $snapname;
+
+	my (undef, $name) = $class->parse_volname($volname);
+
+	my $active_luns = get_active_luns($class, $storeid, $scfg, $name);
+	
+    my $lun = freenas_get_lun_number($scfg, $name);
+	$active_luns->{$lun} = "0:0:0:$lun";
+
+	eval {
+		my $sid = get_sid($scfg, $name);
+		die "activate_volume-> Missing session" if $sid < 0;
+		# Add new LUN's to session
+		os_request("iscsiadm -m session -r $sid -R", 0, 60);
+		sleep 1;
+		# Remove all LUN's from session which is not currently active
+		deactivate_luns($scfg, $name, $active_luns);
+	};
+	die "$@" if $@;
+
+    return 1;
+}
+
+# Procedure for deactivating a LUN:
+#
+# if session exists
+# 	get lun number to deactivate
+#	deactivate lun
+sub deactivate_volume {
+    my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_;
+
+	# TODO: FreeNAS supports exposing a ro LUN from a snapshot
+	# ERROR: Backup of VM xyz failed - unable to activate snapshot from remote zfs storage
+	# at /usr/share/perl5/PVE/Storage/ZFSPlugin.pm line 418.
+	# $vname\@$snapname
+    die "mode failure - unable to deactivate snapshot from remote zfs storage" if $snapname;
+
+	my (undef, $name) = $class->parse_volname($volname);
+
+	my $active_luns = get_active_luns($class, $storeid, $scfg, $name);
+	
+    my $lun = freenas_get_lun_number($scfg, $name);
+	delete $active_luns->{$lun};
+
+	eval {
+		my $sid = get_sid($scfg, $name);
+		die "deactivate_volume-> Missing session" if $sid < 0;
+		deactivate_luns($scfg, $name, $active_luns);
+		delete_session($scfg, $sid) if !%$active_luns;
+	};
+	die $@ if $@;
+
+    return 1;
+}
+
+1;
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index b924f21..23a4aa8 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm FreeNASPlugin.pm
 
 .PHONY: install
 install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 4df6608..d827bbb 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -325,7 +325,7 @@ sub parse_config {
 	    $d->{content} = $def->{content}->[1] if !$d->{content};
 	}
 
-	if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'glusterfs' || $type eq 'zfs' || $type eq 'drbd') {
+	if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'glusterfs' || $type eq 'zfs' || $type eq 'drbd' || $type eq 'freenas') {
 	    $d->{shared} = 1;
 	}
     }
-- 
2.11.0


----

This mail was virus scanned and spam checked before delivery.
This mail is also DKIM signed. See header dkim-signature.





More information about the pve-devel mailing list