[pve-devel] [PATCH] add netapp pnfs cluster-mode plugin
Alexandre Derumier
aderumier at odiso.com
Thu Dec 20 09:09:42 CET 2012
NETAPP CLUSTER MODE, PHYSICAL VIEW
------------------------------------
(Cluster admin VIP)
----------------------------
|------ |INTERCLUSTER NETWORK (10GB)|------
| ---------------------------- |
| | | |
| | | |
[NODE1] [NODE2] [NODE3] [NODE4]
| | | |
---------- ---------- ----------- -----------
AGGREGATE1 AGGREGATE3 AGGREGATE4 AGGREGATE5
[vm1][vm2] [vm3][vm4] [vm5][vm6] [vm7][vm8]
---------- ---------- ----------- ----------
|
----------
AGGREGATE2
[vm9][vm10]
----------
An aggregate in a raid of x disk.
Each aggreagate have differents volumes, 1 by vm.
NETAPP CLUSTER MODE, LOGICAL VIEW
---------------------------------
A virtual server is defined on top of the physical cluster
-----------------------------------
| VSERVER |
| (NFS ips, 1 by node) |
| [vm1][vm2][vm3][vm4][vm5]... |
-----------------------------------
Each node is a pnfs metadatas server.
You only need to mount nfs on 1 metadataserver to access the whole vserver
PNFS VIEW
---------
We only mount the root
mymetadataserver:/ /mnt/pve/netapp-vservername
/
/images/vm1/vm-1-disk-1.raw
/images/vm1/vm-1-disk-2.raw
/images/vm2/
/images/vm3/
/images/vm4/
/images/vm5/
/images/vm6/
/images/vm7/
/images/vm8/
/images/vm9/
/images/vm10/
Eech vm have a volume in the storage.
each node is limited to 500volumes, that why I don't have 1 volume by disk.
We can move volume in realtime in the cluster, from 1 aggreate to another aggregate. (in netapp interface for now)
proxmox storage config
----------------------
netapppnfs: mynetappstorage
path /mnt/pve/netapp-vservername
server mymetadataserver
adminserver X.X.X.X (cluster admin vip)
login clusteradminlogin
password clusteradminpassword
aggregate aggregate1
vserver vservername
options minorversion=1,rw,noatime,nodiratime,noacl,vers=4,rsize=65536,wsize=65536,hard,proto=tcp
content images
maxfiles 1
Signed-off-by: Alexandre Derumier <aderumier at odiso.com>
---
PVE/Storage.pm | 2 +
PVE/Storage/Makefile | 2 +-
PVE/Storage/NetappPNFSPlugin.pm | 555 +++++++++++++++++++++++++++++++++++++++
PVE/Storage/Plugin.pm | 2 +-
4 files changed, 559 insertions(+), 2 deletions(-)
create mode 100644 PVE/Storage/NetappPNFSPlugin.pm
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 17a2377..49568c0 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -25,6 +25,7 @@ use PVE::Storage::RBDPlugin;
use PVE::Storage::SheepdogPlugin;
use PVE::Storage::ISCSIDirectPlugin;
use PVE::Storage::NexentaPlugin;
+use PVE::Storage::NetappPNFSPlugin;
# load and initialize all plugins
PVE::Storage::DirPlugin->register();
@@ -35,6 +36,7 @@ PVE::Storage::RBDPlugin->register();
PVE::Storage::SheepdogPlugin->register();
PVE::Storage::ISCSIDirectPlugin->register();
PVE::Storage::NexentaPlugin->register();
+PVE::Storage::NetappPNFSPlugin->register();
PVE::Storage::Plugin->init();
my $UDEVADM = '/sbin/udevadm';
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index ed65a18..668f937 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm NexentaPlugin.pm NetappPNFSPlugin.pm
.PHONY: install
install:
diff --git a/PVE/Storage/NetappPNFSPlugin.pm b/PVE/Storage/NetappPNFSPlugin.pm
new file mode 100644
index 0000000..a2b79c6
--- /dev/null
+++ b/PVE/Storage/NetappPNFSPlugin.pm
@@ -0,0 +1,555 @@
+package PVE::Storage::NetappPNFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use File::Path;
+use LWP::UserAgent;
+use HTTP::Request;
+use XML::Simple;
+use PVE::Tools qw(run_command);
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+
+use base qw(PVE::Storage::Plugin);
+use Data::Dumper;
+# Netapp helper functions
+
+sub netapp_request {
+ my ($scfg, $vserver, $params) = @_;
+
+ my $vfiler = $vserver ? "vfiler='$vserver'" : "";
+
+ my $content = "<?xml version='1.0' encoding='utf-8' ?>";
+ $content .= "<!DOCTYPE netapp SYSTEM 'file:/etc/netapp_filer.dtd'>";
+ $content .= "<netapp $vfiler version='1.15' xmlns='http://www.netapp.com/filer/admin'>";
+ $content .= $params;
+ $content .= "</netapp>";
+
+ my $url = "http://".$scfg->{adminserver}."/servlets/netapp.servlets.admin.XMLrequest_filer";
+ my $request = HTTP::Request->new('POST',"$url");
+ $request->authorization_basic($scfg->{login},$scfg->{password});
+
+
+ $request->content($content);
+ $request->content_length(length($content));
+
+ my $ua = LWP::UserAgent->new;
+ my $response = $ua->request($request);
+ my $xmlparser = XML::Simple->new( KeepRoot => 1 );
+ my $xmlresponse = $xmlparser->XMLin($response->{_content});
+
+ if(is_array($xmlresponse->{netapp}->{results})){
+ foreach my $result (@{$xmlresponse->{netapp}->{results}}) {
+ if($result->{status} ne 'passed'){
+ die "netapp api error : ".$result->{reason};
+ }
+ }
+ }
+ elsif ($xmlresponse->{netapp}->{results}->{status} ne 'passed') {
+ die "netapp api error : ".$content.$xmlresponse->{netapp}->{results}->{reason};
+ }
+
+ return $xmlresponse;
+}
+
+sub netapp_build_params {
+ my ($execute, %params) = @_;
+
+ my $xml = "<$execute>";
+ while (my ($property, $value) = each(%params)){
+ $xml.="<$property>$value</$property>";
+ }
+ $xml.="</$execute>";
+
+ return $xml;
+
+}
+
+
+sub netapp_create_volume {
+ my ($scfg, $volume, $size) = @_;
+
+ my $aggregate = $scfg->{aggregate};
+ my $xmlparams = netapp_build_params("volume-create", "containing-aggr-name" => $aggregate, "volume" => $volume, "size" => "$size", "junction-path" => "/images/$volume", "space-reserve" => "none");
+ $xmlparams .= netapp_build_params("sis-enable", "path" => "/vol/$volume");
+ $xmlparams .= netapp_build_params("sis-set-config", "enable-compression" => "true", "enable-inline-compression" => "true", "schedule" => "-", "path" => "/vol/$volume");
+ $xmlparams .= netapp_build_params("snapshot-set-reserve", "volume" => $volume, "percentage" => "0");
+ netapp_request($scfg, $scfg->{vserver}, $xmlparams);
+
+}
+
+sub netapp_resize_volume {
+ my ($scfg, $volume, $action, $size) = @_;
+
+ netapp_request($scfg, $scfg->{vserver}, netapp_build_params("volume-size", "volume" => $volume, "new-size" => "$action$size" ));
+
+}
+
+sub netapp_snapshot_create {
+ my ($scfg, $volume, $snapname) = @_;
+
+ netapp_request($scfg, $scfg->{vserver}, netapp_build_params("snapshot-create", "volume" => $volume, "snapshot" => "$snapname" ));
+
+}
+
+sub netapp_snapshot_exist {
+ my ($scfg, $volume, $snap) = @_;
+
+ my $snapshotslist = netapp_request($scfg, $scfg->{vserver}, netapp_build_params("snapshot-list-info", "volume" => "$volume"));
+ my $snapshotexist = undef;
+ $snapshotexist = 1 if (defined($snapshotslist->{"netapp"}->{"results"}->{"snapshots"}->{"snapshot-info"}->{"$snap"}));
+ $snapshotexist = 1 if (defined($snapshotslist->{netapp}->{results}->{"snapshots"}->{"snapshot-info"}->{name}) && $snapshotslist->{netapp}->{results}->{"snapshots"}->{"snapshot-info"}->{name} eq $snap);
+ return $snapshotexist;
+
+}
+
+sub netapp_snapshot_rollback {
+ my ($scfg, $volume, $snapname) = @_;
+
+ netapp_request($scfg, $scfg->{vserver}, netapp_build_params("snapshot-restore-volume", "volume" => $volume, "snapshot" => "$snapname" ));
+
+}
+
+sub netapp_snapshot_delete {
+ my ($scfg, $volume, $snapname) = @_;
+
+ netapp_request($scfg, $scfg->{vserver}, netapp_build_params("snapshot-delete", "volume" => $volume, "snapshot" => "$snapname" ));
+
+}
+
+
+sub netapp_delete_volume {
+ my ($scfg, $volume) = @_;
+
+ my $xmlparams = netapp_build_params("volume-unmount", "volume-name" => "$volume", "force" => "false");
+ $xmlparams .= netapp_build_params("volume-offline", "name" => "$volume");
+ $xmlparams .= netapp_build_params("volume-destroy", "name" => "$volume");
+ netapp_request($scfg, $scfg->{vserver}, $xmlparams);
+
+
+}
+
+sub netapp_aggregate_size {
+ my ($scfg) = @_;
+
+ my $list = netapp_request($scfg, undef, netapp_build_params("aggr-get-iter", "desired-attributes" => "" ));
+
+ foreach my $aggregate (@{$list->{netapp}->{results}->{"attributes-list"}->{"aggr-attributes"}}) {
+ if($aggregate->{"aggregate-name"} eq $scfg->{aggregate}){
+ my $used = $aggregate->{"aggr-space-attributes"}->{"size-used"};
+ my $total = $aggregate->{"aggr-space-attributes"}->{"size-total"};
+ my $free = $aggregate->{"aggr-space-attributes"}->{"size-available"};
+ return ($total, $free, $used, 1);
+ }
+ }
+
+}
+
+
+sub is_array {
+ my ($ref) = @_;
+ # Firstly arrays need to be references, throw
+ # out non-references early.
+ return 0 unless ref $ref;
+
+ # Now try and eval a bit of code to treat the
+ # reference as an array. If it complains
+ # in the 'Not an ARRAY reference' then we're
+ # sure it's not an array, otherwise it was.
+ eval {
+ my $a = @$ref;
+ };
+ if ($@=~/^Not an ARRAY reference/) {
+ return 0;
+ } elsif ($@) {
+ die "Unexpected error in eval: $@\n";
+ } else {
+ return 1;
+ }
+
+}
+
+
+# Configuration
+
+sub type {
+ return 'netapppnfs';
+}
+
+sub plugindata {
+ return {
+ content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1},
+ { images => 1 }],
+ format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ],
+ };
+}
+
+sub properties {
+ return {
+ vserver => {
+ description => "Vserver name",
+ type => 'string',
+ },
+ aggregate => {
+ description => "Aggregate name",
+ type => 'string',
+ },
+ adminserver => {
+ description => "Cluster Management IP or DNS name.",
+ type => 'string', format => 'pve-storage-server',
+ },
+
+ };
+}
+
+sub options {
+ return {
+ path => { fixed => 1 },
+ server => { fixed => 1 },
+ adminserver => { fixed => 1 },
+ login => { fixed => 1 },
+ password => { fixed => 1 },
+ vserver => { fixed => 1 },
+ aggregate => { fixed => 1 },
+ nodes => { optional => 1 },
+ disable => { optional => 1 },
+ maxfiles => { optional => 1 },
+ options => { optional => 1 },
+ content => { optional => 1 },
+ format => { optional => 1 },
+ };
+}
+
+
+sub check_config {
+ my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
+
+ my $vserver = $config->{vserver};
+ $config->{path} = "/mnt/pve/netapp-$vserver" if $create && !$config->{path};
+
+ return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck);
+}
+
+# Storage implementation
+
+sub path {
+ my ($class, $scfg, $volname, $storeid) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $dir = $class->get_subdir($scfg, $vtype);
+
+ $dir .= "/vm$vmid" if $vtype eq 'images';
+
+ my $path = "$dir/$name";
+
+ return wantarray ? ($path, $vmid, $vtype) : $path;
+}
+
+sub parse_volname {
+ my ($class, $volname) = @_;
+
+ if ($volname =~ m!^vm(\d+)/(\S+)$!) {
+ my ($vmid, $name) = ($1, $2);
+ PVE::Storage::Plugin::parse_name_dir($name);
+ return ('images', $name, $vmid);
+ }
+
+ die "unable to parse directory volume name '$volname'\n";
+}
+
+sub list_images {
+ my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_;
+
+ my $imagedir = $class->get_subdir($scfg, 'images');
+
+ my ($defFmt, $vaidFmts) = PVE::Storage::Plugin::default_format($scfg);
+ my $fmts = join ('|', @$vaidFmts);
+
+ my $res = [];
+
+ foreach my $fn (<$imagedir/vm[0-9][0-9]*/*>) {
+ next if $fn !~ m!^(/.+/vm(\d+)/([^/]+\.($fmts)))$!;
+
+ $fn = $1; # untaint
+
+ my $owner = $2;
+ my $name = $3;
+ my $volid = "$storeid:$owner/$name";
+
+ if ($vollist) {
+ my $found = grep { $_ eq $volid } @$vollist;
+ next if !$found;
+ } else {
+ next if defined($vmid) && ($owner ne $vmid);
+ }
+
+ my ($size, $format, $used) = PVE::Storage::Plugin::file_size_info($fn);
+
+ if ($format && $size) {
+ push @$res, {
+ volid => $volid, format => $format,
+ size => $size, vmid => $owner, used => $used };
+ }
+
+ }
+
+ return $res;
+}
+
+
+sub alloc_image {
+ my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_;
+
+ my $imagedir = $class->get_subdir($scfg, 'images');
+ mkpath $imagedir if (-d $imagedir);
+
+ my $volumedir = $imagedir;
+ $volumedir .= "/vm$vmid";
+
+ netapp_create_volume($scfg,"vm$vmid","20 m") if !(-d $volumedir);
+
+ #reading the pnfs root will mount automaticly the volume
+ while(1){
+ sleep 1;
+ my @files = <$imagedir/*>;
+ my $fileexist = undef;
+ foreach (@files) {
+ if ($_ eq $volumedir){
+ $fileexist = 1;
+ last;
+ }
+ }
+ last if $fileexist;
+ }
+
+ if (!$name) {
+ for (my $i = 1; $i < 100; $i++) {
+ my @gr = <$volumedir/vm-$vmid-disk-$i.*>;
+ if (!scalar(@gr)) {
+ $name = "vm-$vmid-disk-$i.$fmt";
+ last;
+ }
+ }
+ }
+
+ die "unable to allocate an image name for VM $vmid in storage '$storeid'\n"
+ if !$name;
+
+ my (undef, $tmpfmt) = PVE::Storage::Plugin::parse_name_dir($name);
+
+ die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n"
+ if $tmpfmt ne $fmt;
+
+ my $path = "$volumedir/$name";
+
+ die "disk image '$path' already exists\n" if -e $path;
+
+ my $cmd = ['/usr/bin/qemu-img', 'create'];
+
+ push @$cmd, '-o', 'preallocation=metadata' if $fmt eq 'qcow2';
+
+ push @$cmd, '-f', $fmt, $path, "${size}K";
+
+ run_command($cmd, errmsg => "unable to create image");
+ #add space to volume
+ netapp_resize_volume($scfg,"vm$vmid","+",$size*1024);
+
+ return "vm$vmid/$name";
+}
+
+sub free_image {
+ my ($class, $storeid, $scfg, $volname) = @_;
+ my $path = $class->path($scfg, $volname);
+
+ if (! -f $path) {
+ die "not exist ".$path;
+ warn "disk image '$path' does not exists\n";
+ } else {
+
+
+ unlink $path;
+ #we can't shrink the volume as the space reclaim is async on file removal
+ #netapp_resize_volume($scfg,"vm$vmid","-",$size);
+
+
+ #now delete netapp volume if they are no more image
+ my ($size, $format, $used) = PVE::Storage::Plugin::file_size_info($path);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $imagedir = $class->get_subdir($scfg, 'images');
+ $imagedir.= "/vm$vmid";
+ my @gr = <$imagedir/vm-$vmid-disk-*>;
+ my $nbdisks = @gr;
+
+ if($nbdisks == 0) {
+
+ netapp_delete_volume($scfg, "vm$vmid");
+
+ my $cmd = ['/bin/umount', $imagedir];
+ run_command($cmd, errmsg => 'umount error');
+ }
+
+ }
+
+ return undef;
+}
+
+
+sub activate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ $cache->{mountdata} = PVE::Storage::NFSPlugin::read_proc_mounts() if !$cache->{mountdata};
+ my $vserver = $scfg->{vserver};
+ my $path = "/mnt/pve/netapp-$vserver";
+ my $server = $scfg->{server};
+ my $export = "/";
+
+ if (!PVE::Storage::NFSPlugin::nfs_is_mounted($server, $export, $path, $cache->{mountdata})) {
+ # NOTE: only call mkpath when not mounted (avoid hang
+ # when NFS server is offline
+ mkpath $path;
+
+ die "unable to activate storage '$storeid' - " .
+ "directory '$path' does not exist\n" if ! -d $path;
+
+ my $options = $scfg->{options} ? $scfg->{options} : "minorversion=1,rw,noatime,nodiratime,noacl,vers=4,rsize=65536,wsize=65536,hard,proto=tcp";
+ PVE::Storage::NFSPlugin::nfs_mount($server, $export, $path, $options);
+ }
+
+ $class->SUPER::activate_storage($storeid, $scfg, $cache);
+}
+
+sub deactivate_storage {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ $cache->{mountdata} = PVE::Storage::NFSPlugin::read_proc_mounts() if !$cache->{mountdata};
+
+ my $path = $scfg->{path};
+ my $server = $scfg->{server};
+ my $export = "/";
+
+ if (PVE::Storage::NFSPlugin::nfs_is_mounted($server, $export, $path, $cache->{mountdata})) {
+ my $cmd = ['/bin/umount', $path];
+ run_command($cmd, errmsg => 'umount error');
+ }
+}
+
+sub activate_volume {
+ my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_;
+ # pnfs will mount automaticly the netapp volume inside the nfs root
+
+}
+
+sub deactivate_volume {
+ my ($class, $storeid, $scfg, $volname, $cache) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ my $imagedir = $class->get_subdir($scfg, 'images');
+ my $volumedir = $imagedir;
+ $volumedir .= "/vm$vmid";
+
+ $cache->{mountdata} = PVE::Storage::NFSPlugin::read_proc_mounts() if !$cache->{mountdata};
+
+ my $server = $scfg->{server};
+ my $export = "/images/vm$vmid";
+
+ if (PVE::Storage::NFSPlugin::nfs_is_mounted($server, $export, $volumedir, $cache->{mountdata})) {
+ my $cmd = ['/bin/umount', $volumedir];
+ run_command($cmd, errmsg => 'umount error');
+ }
+
+}
+
+sub status {
+ my ($class, $storeid, $scfg, $cache) = @_;
+
+ my ($total, $free, $used, $active) = netapp_aggregate_size($scfg);
+
+ return ($total, $free, $used, $active);
+}
+
+
+sub check_connection {
+ my ($class, $storeid, $scfg) = @_;
+
+ my $server = $scfg->{server};
+
+ # test connection to portmapper
+ my $cmd = ['/usr/bin/rpcinfo', '-p', $server];
+
+ eval {
+ run_command($cmd, timeout => 2, outfunc => sub {}, errfunc => sub {});
+ };
+ if (my $err = $@) {
+ return 0;
+ }
+
+ return 1;
+}
+
+sub volume_resize {
+ my ($class, $scfg, $storeid, $volname, $size, $running) = @_;
+
+ die "can't resize this image format" if $volname !~ m/\.(raw)$/;
+
+ my $path = $class->path($scfg, $volname);
+ my ($oldsize, $format, $used) = PVE::Storage::Plugin::file_size_info($path);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ my $incsize = $size - $oldsize;
+ netapp_resize_volume($scfg,"vm$vmid", "+", $incsize);
+
+ return 1 if $running;
+
+ my $cmd = ['/usr/bin/qemu-img', 'resize', $path , $size];
+
+ run_command($cmd, timeout => 10);
+
+ return undef;
+
+}
+
+
+sub volume_snapshot {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ my $path = $class->path($scfg, $volname);
+ my ($size, $format, $used) = PVE::Storage::Plugin::file_size_info($path);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ #don't snasphot twice if multiple disk in one volume
+ netapp_snapshot_create($scfg, "vm$vmid", $snap) if !netapp_snapshot_exist($scfg, "vm$vmid", $snap);
+
+ #reserve space (thinprovisionned) equal to disk size
+ netapp_resize_volume($scfg,"vm$vmid", "+", $size);
+
+ return undef;
+
+}
+
+sub volume_snapshot_rollback {
+ my ($class, $scfg, $storeid, $volname, $snap) = @_;
+
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+ netapp_snapshot_rollback($scfg, "vm$vmid", $snap);
+
+}
+
+sub volume_snapshot_delete {
+ my ($class, $scfg, $storeid, $volname, $snap, $running) = @_;
+
+ my $path = $class->path($scfg, $volname);
+ my ($size, $format, $used) = PVE::Storage::Plugin::file_size_info($path);
+ my ($vtype, $name, $vmid) = $class->parse_volname($volname);
+
+ netapp_snapshot_delete($scfg, "vm$vmid", $snap) if netapp_snapshot_exist($scfg, "vm$vmid", $snap);
+
+ #remove reserved space equal to disk size
+ netapp_resize_volume($scfg,"vm$vmid", "-", $size);
+
+ return undef;
+}
+
+1;
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 886117b..c7cf9a6 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -300,7 +300,7 @@ sub parse_config {
$d->{content} = $def->{content}->[1] if !$d->{content};
}
- if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' ) {
+ if ($type eq 'iscsi' || $type eq 'nfs' || $type eq 'rbd' || $type eq 'sheepdog' || $type eq 'iscsidirect' || $type eq 'nexenta' || $type eq 'netapppnfs') {
$d->{shared} = 1;
}
}
--
1.7.10.4
More information about the pve-devel
mailing list