[pve-devel] [PATCH] Added support for ZFS Storage Plugin
Alexandre Derumier
aderumier at odiso.com
Wed Oct 2 04:58:10 CEST 2013
From: Michael Rasmussen <mir at datanom.net>
example of storage.cfg
zfs: omnios
blocksize 8k
target iqn.2010-09.org.openindiana:target1
pool pool1
iscsiprovider comstar
portal 192.168.0.1
sudo 1 (optionnal)
content images
note for fast ssh login:
on solaris host :
/etc/ssh/sshd_config
LookupClientHostnames no
VerifyReverseMapping no
GSSAPIAuthentication no
note for nexenta:
rm /root/.bash_profile
to avoid to go in nmc console by default
Signed-off-by: Michael Rasmussen <mir at datanom.net>
Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
PVE/Storage.pm | 2 +
PVE/Storage/Makefile | 2 +-
PVE/Storage/Plugin.pm | 2 +-
PVE/Storage/ZFSPlugin.pm | 629 ++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 633 insertions(+), 2 deletions(-)
create mode 100644 PVE/Storage/ZFSPlugin.pm
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 7858f41..cea40f1 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -28,6 +28,7 @@ use PVE::Storage::SheepdogPlugin;
use PVE::Storage::ISCSIDirectPlugin;
use PVE::Storage::NexentaPlugin;
use PVE::Storage::GlusterfsPlugin;
+use PVE::Storage::ZFSPlugin;
# load and initialize all plugins
PVE::Storage::DirPlugin->register();
@@ -39,6 +40,7 @@ PVE::Storage::SheepdogPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
PVE::Storage::NexentaPlugin->register();
PVE::Storage::GlusterfsPlugin->register();
+PVE::Storage::ZFSPlugin->register();
PVE::Storage::Plugin->init();
my $UDEVADM = '/sbin/udevadm';
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index 0014c51..f59f2d2 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm GlusterfsPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm GlusterfsPlugin.pm ZFSPlugin.pm
.PHONY: install
install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 59f4a8e..cb4ecbf 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -312,7 +312,7 @@ sub parse_config {
$d->{content} = $def->{content}->[1] if !$d->{content};
}
- if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' || $type eq 'glusterfs') {
+ if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' || $type eq 'glusterfs' || $type eq 'zfs') {
$d->{shared} = 1;
}
}
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
new file mode 100644
index 0000000..6cc0229
--- /dev/null
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -0,0 +1,629 @@
+package PVE::Storage::ZFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use POSIX;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+use Digest::MD5 qw(md5_hex);
+
+use base qw(PVE::Storage::Plugin);
+
+my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+
+sub zfs_request {
+ my ($scfg, $timeout, $method, @params) = @_;
+
+ my $cmdmap = ();
+ my $msg = '';
+ my $zfscmd;
+ my $target;
+ $timeout = 5 if !$timeout;
+
+ my $output = sub {
+ my $line = shift;
+ $msg .= "$line\n";
+ };
+
+ if ($scfg->{iscsiprovider} eq 'comstar') {
+ my $stmfadmcmd = "/usr/sbin/stmfadm";
+ my $sbdadmcmd = "/usr/sbin/sbdadm";
+
+ $cmdmap = {
+ create_lu => { cmd => $stmfadmcmd, method => 'create-lu' },
+ delete_lu => { cmd => $stmfadmcmd, method => 'delete-lu' },
+ import_lu => { cmd => $stmfadmcmd, method => 'import-lu' },
+ modify_lu => { cmd => $stmfadmcmd, method => 'modify-lu' },
+ add_view => { cmd => $stmfadmcmd, method => 'add-view' },
+ list_view => { cmd => $stmfadmcmd, method => 'list-view' },
+ list_lu => { cmd => $sbdadmcmd, method => 'list-lu' },
+ zpool_list => { cmd => 'zpool', method => 'list' },
+ };
+ } else {
+ die 'unknown iscsi provider. Available [comstar]';
+ }
+
+ if ($cmdmap->{$method}) {
+ $zfscmd = $cmdmap->{$method}->{cmd};
+ $method = $cmdmap->{$method}->{method};
+ } else {
+ $zfscmd = 'zfs';
+ }
+
+
+ if ($scfg->{sudo}) {
+ $zfscmd = 'sudo ' . $zfscmd;
+ $target = $scfg->{portal};
+ } else {
+ $target = 'root@' . $scfg->{portal};
+ }
+
+ my $cmd = [@ssh_cmd, $target, $zfscmd, $method, @params];
+
+ run_command($cmd, outfunc => $output, timeout => $timeout);
+
+ return $msg;
+}
+
+sub zfs_parse_size {
+ my ($text) = @_;
+
+ return 0 if !$text;
+
+ if ($text =~ m/^(\d+(\.\d+)?)([TGMK])?$/) {
+ my ($size, $reminder, $unit) = ($1, $2, $3);
+ return $size if !$unit;
+ if ($unit eq 'K') {
+ $size *= 1024;
+ } elsif ($unit eq 'M') {
+ $size *= 1024*1024;
+ } elsif ($unit eq 'G') {
+ $size *= 1024*1024*1024;
+ } elsif ($unit eq 'T') {
+ $size *= 1024*1024*1024*1024;
+ }
+
+ if ($reminder) {
+ $size = ceil($size);
+ }
+ return $size;
+ } else {
+ return 0;
+ }
+}
+
+sub zfs_get_pool_stats {
+ my ($scfg) = @_;
+
+ my $size = 0;
+ my $used = 0;
+
+ my $text = zfs_request($scfg, undef, 'get', '-o', 'value', '-Hp',
+ 'available,used', $scfg->{pool});
+
+ my @lines = split /\n/, $text;
+
+ if($lines[0] =~ /^(\d+)$/) {
+ $size = $1;
+ }
+
+ if($lines[1] =~ /^(\d+)$/) {
+ $used = $1;
+ }
+
+ return ($size, $used);
+}
+
+sub zfs_parse_zvol_list {
+ my ($text) = @_;
+
+ my $list = ();
+
+ return $list if !$text;
+
+ my @lines = split /\n/, $text;
+ foreach my $line (@lines) {
+ if ($line =~ /^(.+)\s+([a-zA-Z0-9\.]+|\-)\s+(.+)$/) {
+ my $zvol = {};
+ my $name;
+ my $disk;
+ my @zvols = split /\//, $1;
+ my $pool = $zvols[0];
+
+ if (scalar(@zvols) == 2 && $zvols[0] !~ /^rpool$/) {
+ $disk = $zvols[1];
+ next unless $disk =~ m!^(\w+)-(\d+)-(\w+)-(\d+)$!;
+ $name = $pool . '/' . $disk;
+ }
+ else {
+ next;
+ }
+
+ $zvol->{name} = $name;
+ $zvol->{size} = zfs_parse_size($2);
+ if ($3 !~ /^-$/) {
+ $zvol->{origin} = $3;
+ }
+ push @$list, $zvol;
+ }
+ }
+
+ return $list;
+}
+
+sub zfs_get_lu_name {
+ my ($scfg, $zvol) = @_;
+ my $object;
+
+ if ($zvol =~ /^.+\/.+/) {
+ $object = "/dev/zvol/rdsk/$zvol";
+ }
+ else {
+ $object = "/dev/zvol/rdsk/$scfg->{pool}/$zvol";
+ }
+
+ my $text = zfs_request($scfg, undef, 'list_lu');
+ my @lines = split /\n/, $text;
+ foreach my $line (@lines) {
+ return $1 if ($line =~ /(\w+)\s+\d+\s+$object$/);
+ }
+ die "Could not find lu_name for zvol $zvol";
+}
+
+sub zfs_get_zvol_size {
+ my ($scfg, $zvol) = @_;
+
+ my $text = zfs_request($scfg, undef, 'get', '-Hp', 'volsize', "$scfg->{pool}/$zvol");
+
+ if($text =~ /volsize\s(\d+)/){
+ return $1;
+ }
+
+ die "Could not get zvol size";
+}
+
+sub zfs_add_lun_mapping_entry {
+ my ($scfg, $zvol, $guid) = @_;
+
+ if (! defined($guid)) {
+ $guid = zfs_get_lu_name($scfg, $zvol);
+ }
+
+ zfs_request($scfg, undef, 'add_view', $guid);
+}
+
+sub zfs_delete_lu {
+ my ($scfg, $zvol) = @_;
+
+ my $guid = zfs_get_lu_name($scfg, $zvol);
+
+ zfs_request($scfg, undef, 'delete_lu', $guid);
+}
+
+sub zfs_create_lu {
+ my ($scfg, $zvol) = @_;
+
+ my $prefix = '600144f';
+ my $digest = md5_hex($zvol);
+ $digest =~ /(\w{7}(.*))/;
+ my $guid = "$prefix$2";
+
+ zfs_request($scfg, undef, 'create_lu', '-p', 'wcd=false', '-p', "guid=$guid", "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+
+ return $guid;
+}
+
+sub zfs_import_lu {
+ my ($scfg, $zvol) = @_;
+
+ zfs_request($scfg, undef, 'import_lu', "/dev/zvol/rdsk/$scfg->{pool}/$zvol");
+}
+
+sub zfs_resize_lu {
+ my ($scfg, $zvol, $size) = @_;
+
+ my $guid = zfs_get_lu_name($scfg, $zvol);
+
+ zfs_request($scfg, undef, 'modify_lu', '-s', "${size}K", $guid);
+}
+
+sub zfs_create_zvol {
+ my ($scfg, $zvol, $size) = @_;
+
+ zfs_request($scfg, undef, 'create', '-b', $scfg->{blocksize}, '-V', "${size}k", "$scfg->{pool}/$zvol");
+}
+
+sub zfs_delete_zvol {
+ my ($scfg, $zvol) = @_;
+
+ zfs_request($scfg, undef, 'destroy', '-r', "$scfg->{pool}/$zvol");
+}
+
+sub zfs_get_lun_number {
+ my ($scfg, $guid) = @_;
+ my $lunnum = undef;
+
+ die "could not find lun_number for guid $guid" if !$guid;
+
+ my $text = zfs_request($scfg, undef, 'list_view', '-l', $guid);
+ my @lines = split /\n/, $text;
+ foreach my $line (@lines) {
+ if ($line =~ /^\s*LUN\s*:\s*(\d+)$/) {
+ $lunnum = $1;
+ last;
+ }
+ }
+
+ return $lunnum;
+}
+
+sub zfs_list_zvol {
+ my ($scfg) = @_;
+
+ my $text = zfs_request($scfg, 10, 'list', '-o', 'name,volsize,origin', '-Hr');
+ my $zvols = zfs_parse_zvol_list($text);
+ return undef if !$zvols;
+
+ my $list = ();
+ foreach my $zvol (@$zvols) {
+ my @values = split('/', $zvol->{name});
+
+ my $pool = $values[0];
+ my $image = $values[1];
+ my $owner;
+ if ($image =~ m/^((vm|base)-(\d+)-\S+)$/) {
+ $owner = $3;
+ }
+ else {
+ next;
+ }
+
+ my $parent = $zvol->{origin};
+ if($zvol->{origin} && $zvol->{origin} =~ m/^$scfg->{pool}\/(\S+)$/){
+ $parent = $1;
+ }
+
+ $list->{$pool}->{$image} = {
+ name => $image,
+ size => $zvol->{size},
+ parent => $parent,
+ format => 'raw',
+ vmid => $owner
+ };
+ }
+
+ return $list;
+}
+
+# Configuration
+
+sub type {
+ return 'zfs';
+}
+
+sub plugindata {
+ return {
+ content => [ {images => 1}, { images => 1 }],
+ };
+}
+
+sub properties {
+ return {
+ chap => {
+ description => "chap",
+ type => 'string',
+ },
+ pwd => {
+ description => "password",
+ type => 'string',
+ },
+ iscsiprovider => {
+ description => "iscsi provider",
+ type => 'string',
+ },
+ sudo => {
+ description => "use sudo",
+ type => 'boolean',
+ },
+ };
+}
+
+sub options {
+ return {
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ portal => { fixed => 1 },
+ target => { fixed => 1 },
+ pool => { fixed => 1 },
+ chap => { optional => 1 },
+ pwd => { optional => 1 },
+ blocksize => { fixed => 1 },
+ iscsiprovider => { fixed => 1 },
+ sudo => { optional => 1 },
+ content => { optional => 1 },
+ };
+}
+
+# Storage implementation
+
+sub parse_volname {
+ my ($class, $volname) = @_;
+
+ if ($volname =~ m/^(((base|vm)-(\d+)-\S+)\/)?((base)?(vm)?-(\d+)-\S+)$/) {
+ return ('images', $5, $8, $2, $4, $6);
+ }
+
+ die "unable to parse zfs volume name '$volname'\n";
+}
+
+sub path {
+ my ($class, $scfg, $volname) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $target = $scfg->{target};
+ my $portal = $scfg->{portal};
+
+ my $guid = zfs_get_lu_name($scfg, $name);
+ my $lun = zfs_get_lun_number($scfg, $guid);
+
+ my $path = "iscsi://$portal/$target/$lun";
+
+ return ($path, $vmid, $vtype);
+}
+
+my $find_free_diskname = sub {
+ my ($storeid, $scfg, $vmid) = @_;
+
+ my $name = undef;
+ my $volumes = zfs_list_zvol($scfg);
+
+ my $disk_ids = {};
+ my $dat = $volumes->{$scfg->{pool}};
+
+ foreach my $image (keys %$dat) {
+ my $volname = $dat->{$image}->{name};
+ if ($volname =~ m/(vm|base)-$vmid-disk-(\d+)/){
+ $disk_ids->{$2} = 1;
+ }
+ }
+
+ for (my $i = 1; $i < 100; $i++) {
+ if (!$disk_ids->{$i}) {
+ return "vm-$vmid-disk-$i";
+ }
+ }
+
+ die "unable to allocate an image name for VM $vmid in storage '$storeid'\n";
+};
+
+sub create_base {
+ my ($class, $storeid, $scfg, $volname) = @_;
+
+ my $snap = '__base__';
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ die "create_base not possible with base image\n" if $isBase;
+
+ my $newname = $name;
+ $newname =~ s/^vm-/base-/;
+
+ my $newvolname = $basename ? "$basename/$newname" : "$newname";
+
+ zfs_delete_lu($scfg, $name);
+ zfs_request($scfg, undef, 'rename', "$scfg->{pool}/$name", "$scfg->{pool}/$newname");
+
+ my $guid = zfs_create_lu($scfg, $newname);
+ zfs_add_lun_mapping_entry($scfg, $newname, $guid);
+
+ my $running = undef; #fixme : is create_base always offline ?
+
+ $class->volume_snapshot($scfg, $storeid, $newname, $snap, $running);
+
+ return $newvolname;
+}
+
+sub clone_image {
+ my ($class, $scfg, $storeid, $volname, $vmid) = @_;
+
+ my $snap = '__base__';
+
+ my ($vtype, $basename, $basevmid, undef, undef, $isBase) =
+ $class->parse_volname($volname);
+
+ die "clone_image only works on base images\n" if !$isBase;
+
+ my $name = &$find_free_diskname($storeid, $scfg, $vmid);
+
+ warn "clone $volname: $basename to $name\n";
+
+ zfs_request($scfg, undef, 'clone', "$scfg->{pool}/$basename\@$snap", "$scfg->{pool}/$name");
+
+ my $guid = zfs_create_lu($scfg, $name);
+ zfs_add_lun_mapping_entry($scfg, $name, $guid);
+
+ return $name;
+}
+
+sub alloc_image {
+ my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+
+ die "unsupported format '$fmt'" if $fmt ne 'raw';
+
+ die "illegal name '$name' - sould be 'vm-$vmid-*'\n"
+ if $name && $name !~ m/^vm-$vmid-/;
+
+ $name = &$find_free_diskname($storeid, $scfg, $vmid);
+
+ zfs_create_zvol($scfg, $name, $size);
+ my $guid = zfs_create_lu($scfg, $name);
+ zfs_add_lun_mapping_entry($scfg, $name, $guid);
+
+ return $name;
+}
+
+sub free_image {
+ my ($class, $storeid, $scfg, $volname, $isBase) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ zfs_delete_lu($scfg, $name);
+ eval {
+ zfs_delete_zvol($scfg, $name);
+ };
+ do {
+ my $err = $@;
+ my $guid = zfs_create_lu($scfg, $name);
+ zfs_add_lun_mapping_entry($scfg, $name, $guid);
+ die $err;
+ } if $@;
+
+ return undef;
+}
+
+sub list_images {
+ my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+ $cache->{zfs} = zfs_list_zvol($scfg) if !$cache->{zfs};
+ my $zfspool = $scfg->{pool};
+ my $res = [];
+
+ if (my $dat = $cache->{zfs}->{$zfspool}) {
+
+ foreach my $image (keys %$dat) {
+
+ my $volname = $dat->{$image}->{name};
+ my $parent = $dat->{$image}->{parent};
+
+ my $volid = undef;
+ if ($parent && $parent =~ m/^(\S+)@(\S+)$/) {
+ my ($basename) = ($1);
+ $volid = "$storeid:$basename/$volname";
+ } else {
+ $volid = "$storeid:$volname";
+ }
+
+ my $owner = $dat->{$volname}->{vmid};
+ if ($vollist) {
+ my $found = grep { $_ eq $volid } @$vollist;
+ next if !$found;
+ } else {
+ next if defined ($vmid) && ($owner ne $vmid);
+ }
+
+ my $info = $dat->{$volname};
+ $info->{volid} = $volid;
+ push @$res, $info;
+ }
+ }
+
+ return $res;
+}
+
+sub status {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ my $total = 0;
+ my $free = 0;
+ my $used = 0;
+ my $active = 0;
+
+ eval {
+ ($total, $used) = zfs_get_pool_stats($scfg);
+ $active = 1;
+ $free = $total - $used;
+ };
+ warn $@ if $@;
+
+ return ($total, $free, $used, $active);
+}
+
+sub activate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ return 1;
+}
+
+sub deactivate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+ return 1;
+}
+
+sub activate_volume {
+ my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+ return 1;
+}
+
+sub deactivate_volume {
+ my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+ return 1;
+}
+
+sub volume_size_info {
+ my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+ return zfs_get_zvol_size($scfg, $volname);
+}
+
+sub volume_resize {
+ my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+ my $new_size = ($size/1024);
+
+ zfs_request($scfg, undef, 'set', 'volsize=' . $new_size . 'k', "$scfg->{pool}/$volname");
+ zfs_resize_lu($scfg, $volname, $new_size);
+}
+
+sub volume_snapshot {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ zfs_request($scfg, undef, 'snapshot', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_snapshot_rollback {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ zfs_delete_lu($scfg, $volname);
+
+ zfs_request($scfg, undef, 'rollback', "$scfg->{pool}/$volname\@$snap");
+
+ zfs_import_lu($scfg, $volname);
+
+ zfs_add_lun_mapping_entry($scfg, $volname);
+}
+
+sub volume_snapshot_delete {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ zfs_request($scfg, undef, 'destroy', "$scfg->{pool}/$volname\@$snap");
+}
+
+sub volume_has_feature {
+ my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_;
+
+ my $features = {
+ snapshot => { current => 1, snap => 1},
+ clone => { base => 1},
+ template => { current => 1},
+ copy => { base => 1, current => 1},
+ };
+
+ my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
+ $class->parse_volname($volname);
+
+ my $key = undef;
+ if($snapname){
+ $key = 'snap';
+ } else {
+ $key = $isBase ? 'base' : 'current';
+ }
+ return 1 if $features->{$feature}->{$key};
+
+ return undef;
+}
+
+1;
--
1.7.10.4
More information about the pve-devel
mailing list