[pve-devel] [PATCH storage 1/2] add SCST provider to ZFS over iSCSI storage type
Dzmitry Kotsikau
dkotsikau at gmail.com
Thu May 3 14:50:21 CEST 2018
Signed-off-by: Dzmitry Kotsikau <dkotsikau at gmail.com>
---
PVE/Storage/LunCmd/Makefile | 2 +-
PVE/Storage/LunCmd/Scst.pm | 557 ++++++++++++++++++++++++++++++++++++++++++++
PVE/Storage/ZFSPlugin.pm | 15 +-
3 files changed, 570 insertions(+), 4 deletions(-)
create mode 100644 PVE/Storage/LunCmd/Scst.pm
diff --git a/PVE/Storage/LunCmd/Makefile b/PVE/Storage/LunCmd/Makefile
index b959255..5a44cbd 100644
--- a/PVE/Storage/LunCmd/Makefile
+++ b/PVE/Storage/LunCmd/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Comstar.pm Istgt.pm Iet.pm
+SOURCES=Comstar.pm Istgt.pm Iet.pm Scst.pm
.PHONY: install
install:
diff --git a/PVE/Storage/LunCmd/Scst.pm b/PVE/Storage/LunCmd/Scst.pm
new file mode 100644
index 0000000..4e0a67b
--- /dev/null
+++ b/PVE/Storage/LunCmd/Scst.pm
@@ -0,0 +1,557 @@
+package PVE::Storage::LunCmd::Scst;
+
+#1) Install scst on proxmox node
+
+#apt-get install git gawk build-essential flex bison automake autoconf pve-headers dkms
+#git clone https://github.com/bvanassche/scst.git
+#cd scst
+#make scst scst_local iscsi-scst scst srpt usr
+#make scst_install scst_local_install iscsi_install scst_install srpt_install usr_install
+#depmod -a
+#cat > /etc/default/scst << 'EOF'
+#ISCSID_OPTIONS="-a 192.168.254.10 -u0 -g0 -p3260"
+#SCST_TARGET_MODULES="scst scst_vdisk iscsi-scst"
+#EOF
+
+#2) Create portal
+
+#cat > /etc/scst.conf <'EOF'
+#TARGET_DRIVER iscsi {
+# DefaultTime2Wait 2
+# DefaultTime2Retain 90
+# enabled 1
+#}
+#EOF
+
+
+
+
+use strict;
+use warnings;
+use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach);
+use Data::Dumper;
+
+sub get_base;
+
+# A logical unit can max have 16383 LUNs
+my $MAX_LUNS = 16383;
+my $HANDLER = 'vdisk_fileio';
+
+my $CONFIG_FILE = '/etc/scst.conf';
+my $CONFIG_FILE_TMP = '/tmp/pve-scst.conf';
+
+my $DAEMON = '/usr/local/sbin/iscsi-scstd';
+my $SETTINGS = undef;
+my $CONFIG;
+
+my @ssh_opts = ('-o', 'BatchMode=yes', '-o','PreferredAuthentications=publickey');
+my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
+my @scp_cmd = ('/usr/bin/scp', @ssh_opts);
+my $id_rsa_path = '/etc/pve/priv/zfs';
+my $scstadmin = '/usr/local/sbin/scstadmin';
+
+my $execute_command = sub {
+ my ($scfg, $exec, $timeout, $method, @params) = @_;
+
+ my $msg = '';
+ my $err = undef;
+ my $target;
+ my $cmd;
+ my $res = ();
+
+ $timeout = 10 if !$timeout;
+
+ my $output = sub {
+ my $line = shift;
+ $msg .= "$line\n";
+ };
+
+ my $errfunc = sub {
+ my $line = shift;
+ $err .= "$line";
+ };
+
+ if ($exec eq 'scp') {
+ $target = 'root@[' . $scfg->{portal} . ']';
+ $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"];
+ } else {
+ $target = 'root@' . $scfg->{portal};
+ $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params];
+ }
+
+ eval {
+ run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout);
+ };
+ if ($@) {
+ $res = {
+ result => 0,
+ msg => $err,
+ }
+ } else {
+ $res = {
+ result => 1,
+ msg => $msg,
+ }
+ }
+
+ return $res;
+};
+
+my $update_config = sub {
+ my ($scfg) = @_;
+
+ my @params = ('-write_config ', $CONFIG_FILE);
+ my $res = $execute_command->($scfg, 'ssh', undef, $scstadmin, @params);
+ die $res->{msg} unless $res->{result};
+};
+
+my $get_target_tid = sub {
+ my ($scfg) = @_;
+
+ my $res = {msg => undef };
+
+ my @params = ("/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/tid");
+ $res = $execute_command->($scfg, 'ssh', 10, 'test', '-f', @params);
+ die "The target not found! ". $res->{msg} unless $res->{result};
+
+ my $tid = undef;
+
+ $res = $execute_command->($scfg, 'ssh', undef, 'cat', @params);
+ die $res->{msg} unless $res->{result};
+ my @lines = split "\n", $res->{msg};
+ $tid = $lines[0];
+ return $tid;
+};
+
+
+sub parseLine {
+ my $line = shift;
+ my $hash = shift;
+ my $child = shift;
+ return if (! $line);
+ return if ($line =~ /^\s*$/);
+
+ $line =~ s/^\s+//; $line =~ s/\s+$//;
+
+ my @elements;
+ while ($line =~ m/"([^"\\]*(\\.[^"\\]*)*)"|([^\s]+)/g) {
+ push @elements, defined($1) ? $1:$3;
+ }
+
+ my $attribute = $elements[0];
+ my $value = $elements[1];
+ my $value2 = $elements[2];
+
+ if (defined($attribute) && defined($value) && defined($value2)) {
+ $$hash{$attribute}->{$value}->{$value2} = $child;
+ } elsif (defined($attribute) && defined($value)) {
+ $$hash{$attribute}->{$value} = $child;
+ } elsif (defined($attribute)) {
+ $$hash{$attribute} = $child;
+ }
+};
+
+
+sub parseStanza {
+ my $buffer = shift;
+ my $line;
+ my %hash;
+ my $attribute;
+ my $value;
+ my $value2;
+ my $quoted;
+
+ while ($#{$buffer} > -1) {
+ my $char = shift @{$buffer};
+
+ if ($char eq '{') {
+ my $child = parseStanza($buffer);
+ if ($line) {
+ parseLine($line, \%hash, $child);
+ $line = undef;
+ }
+ next;
+ }
+
+ if ($char eq '}') {
+ return \%hash
+ };
+
+ if ($char eq "\n") {
+ my %empty;
+ parseLine($line, \%hash, \%empty);
+ $line = undef;
+ } else {
+ $line .= $char;
+ }
+ }
+ return \%hash;
+};
+
+sub parseLuns {
+ my $scfg = shift;
+ my $luns = shift;
+ my $h_luns = ();
+ my $base = get_base;
+
+ foreach my $lun (keys %{$$luns} ) {
+ foreach my $device (keys %{$$luns->{$lun}}) {
+ if (!defined($$CONFIG{'HANDLER'}) ||
+ !defined($$CONFIG{'HANDLER'}->{$HANDLER}) ||
+ !defined($$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device})) {
+ die "Wrong configuration file $CONFIG_FILE!";
+ }
+ my $dev = $$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device};
+
+ my $conf = undef;
+ $conf->{Device} = (keys %{$$dev{'prod_id'}})[0];
+ $conf->{Path} = (keys %{$$dev{'filename'}})[0];
+ $conf->{blocksize} = (keys %{$$dev{'blocksize'}})[0];
+ $conf->{size} = (keys %{$$dev{'size'}})[0];
+ if ($conf->{Path} && $conf->{Path} =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) {
+ $conf->{include} = 1;
+ } else {
+ $conf->{include} = 0;
+ }
+ $conf->{lun} = $lun;
+ push @{$h_luns}, $conf;
+ }
+ }
+ return $h_luns;
+}
+
+my $readConfigFile = sub {
+ my ($scfg) = @_;
+ my $buffer;
+ my @stanza;
+ my $level;
+ my $used = ();
+
+ my @commands = ("touch '$CONFIG_FILE_TMP'",
+ "chmod 600 '$CONFIG_FILE_TMP'",
+ "$scstadmin -write_config '$CONFIG_FILE_TMP' -nonkey",
+ "cat '$CONFIG_FILE_TMP'",
+ "rm -f '$CONFIG_FILE_TMP'"
+ );
+ my $res = $execute_command->($scfg, 'ssh', 20, join('&&', @commands));
+ die $res->{msg} unless $res->{result};
+ my @lines = split "\n", $res->{msg};
+ foreach my $line (@lines){
+ $line =~ s/^\#.*//;
+ $line =~ s/[^\\]\#.*//;
+ $line =~ s/\\(.)/$1/g;
+ $buffer .= $line."\n";
+ }
+ my @buff_a;
+ @buff_a = split(//, $buffer);
+ $CONFIG = parseStanza(\@buff_a);
+
+ if (!defined($$CONFIG{'TARGET_DRIVER'}) ||
+ !(scalar keys %{$$CONFIG{'TARGET_DRIVER'}}) ||
+ !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}) ||
+ !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}})
+ )
+ {
+ die Dumper($CONFIG) ."\nWrong configuration file $CONFIG_FILE!";
+
+ }
+
+ $SETTINGS->{target} = $scfg->{target};
+ my $tgt = $$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}};
+ if (defined($$tgt{'LUN'})) {
+ push @{$SETTINGS->{luns}}, @{parseLuns($scfg ,\$tgt->{'LUN'})};
+ }
+
+ if (defined($$tgt{'GROUP'})) {
+ foreach my $group (keys %{$$tgt{'GROUP'}}) {
+ if (defined($$tgt{'GROUP'}->{$group}->{'LUN'})) {
+ push @{$SETTINGS->{luns}}, @{parseLuns($scfg , \$tgt->{'GROUP'}->{$group}->{'LUN'} )};
+ }
+ }
+ }
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ $used->{$lun->{lun}} = 1;
+ }
+
+ $SETTINGS->{used} = $used;
+# die Dumper($SETTINGS);
+ return 0;
+};
+
+my $get_lu_name = sub {
+ my $i;
+
+ my $used = $SETTINGS->{used};
+ for ($i = 0; $i < $MAX_LUNS; $i++) {
+ last unless $used->{$i};
+ }
+ $SETTINGS->{used}->{$i} = 1;
+
+ return $i;
+};
+
+my $init_lu_name = sub {
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ $SETTINGS->{used}->{$lun->{lun}} = 1;
+ }
+};
+
+my $free_lu_name = sub {
+ my ($lu_name) = @_;
+ my $new;
+
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ if ($lun->{lun} != $lu_name) {
+ push @$new, $lun;
+ }
+ }
+
+ $SETTINGS->{luns} = $new;
+ $SETTINGS->{used}->{$lu_name} = 0;
+};
+
+my $make_lun = sub {
+ my ($scfg, $path) = @_;
+
+ die 'Maximum number of LUNs per target is 16383' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS;
+
+ my $lun = $get_lu_name->();
+ my $conf = {
+ lun => $lun,
+ Path => $path,
+ include => 1,
+ };
+ push @{$SETTINGS->{luns}}, $conf;
+ $SETTINGS->{used}->{$lun} = 1;
+ # print Dumper ($SETTINGS);
+ return $conf;
+};
+
+my $list_view = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+ my $lun = undef;
+
+ my $object = $params[0];
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ next unless $lun->{include} == 1;
+ if ($lun->{Path} =~ /^$object$/) {
+ return $lun->{lun} if (defined($lun->{lun}));
+ die "$lun->{Path}: Missing LUN";
+ }
+ }
+
+ return $lun;
+};
+
+my $list_lun = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+ my $name = undef;
+
+ my $object = $params[0];
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ next unless $lun->{include} == 1;
+ if ($lun->{Path} =~ /^$object$/) {
+ return $lun->{Path};
+ }
+ }
+ return $name;
+};
+
+my $parse_size = sub {
+ my ($text) = @_;
+
+ return 0 if !$text;
+
+ if ($text =~ m/^(\d+(\.\d+)?)(k)?$/) {
+ my ($size, $reminder, $unit) = ($1, $2, $3);
+ return $size if !$unit;
+ if ($unit eq 'k') {
+ $size *= 1024;
+ }
+ if ($reminder) {
+ $size = ceil($size);
+ }
+ return $size;
+ } else {
+ return 0;
+ }
+};
+
+sub append_group_params {
+ my $scfg = shift;
+ my $params = shift;
+ if ($scfg->{comstar_tg}) {
+ push @{$params}, ('-group', $scfg->{comstar_tg} );
+ }
+}
+
+my $create_lun = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+
+ if ($list_lun->($scfg, $timeout, $method, @params)) {
+ die "$params[0]: LUN exists";
+ }
+ my $lun = $params[0];
+ $lun = $make_lun->($scfg, $lun);
+
+ my $tid = $get_target_tid->($scfg);
+ my $device = 'diskt'.$tid.'l'.$lun->{lun};
+ print "Create $HANDLER device $device using filename=$lun->{Path}...";
+ my $blocksize = $parse_size->($scfg->{blocksize});
+ my $res = {msg => undef};
+
+
+ my @commands = (
+ "(i=1; while [ ! -e '$lun->{Path}' -a \$i -le 20 ]; do sleep 1; i=\$((i+1)); done)",
+ "[ -e '$lun->{Path}' ]",
+ "$scstadmin");
+ @params = ('-open_dev', $device, '-handler', $HANDLER, '-attributes', "filename=$lun->{Path},nv_cache=1,blocksize=512,thin_provisioned=1,zero_copy=1");
+
+ $res = $execute_command->($scfg, 'ssh', 30, join ('&&', @commands), @params);
+# $res = $execute_command->($scfg, 'ssh', 30, $scstadmin, @params);
+ do {
+ $free_lu_name->($lun->{lun});
+ if ($res->{msg}) {
+ die $res->{msg} ;
+ } else {
+ die 'create_lun: timeout or unknown error! command: '. join ('&&', @commands) . ' ' . join ' ' , @params;
+ }
+ } unless $res->{result};
+
+ print "Done!\n";
+ print "Create LUN $lun->{lun} using the device $device for target $scfg->{target}...";
+
+ @params = ('-add_lun', $lun->{lun}, '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}");
+ append_group_params ($scfg, \@params);
+
+ $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+
+ do {
+ print "Failed!\nRemoving device...";
+ @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt');
+ $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+ print "Done!\n";
+ $free_lu_name->($lun->{lun});
+ $update_config->($scfg);
+ die $res->{msg};
+ } unless $res->{result};
+ print "Done!\n";
+ $update_config->($scfg);
+ return $res->{msg};
+};
+
+my $delete_lun = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+ my $res = {msg => undef};
+
+ my $path = $params[0];
+
+
+ foreach my $lun (@{$SETTINGS->{luns}}) {
+ if ($lun->{Path} eq $path) {
+
+ my $tid = $get_target_tid->($scfg);
+ my $device = 'diskt'.$tid.'l'.$lun->{lun};
+ print "Delete $HANDLER device $device using filename=$lun->{Path}...";
+
+ @params = ('-rem_lun', $lun->{lun}, '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}", '-noprompt');
+ append_group_params ($scfg, \@params);
+
+ $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+ do {
+ $free_lu_name->($lun->{lun});
+ die $res->{msg};
+ } unless $res->{result};
+
+ print "Done!\n";
+ print "Delete LUN $lun->{lun} using the device $device for target $scfg->{target}...";
+
+ @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt');
+ $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+
+ if ($res->{result}) {
+ $free_lu_name->($lun->{lun});
+ print "Done!\n";
+ last;
+ } else {
+ $update_config->($scfg);
+ die $res->{msg};
+ }
+ }
+ }
+ $update_config->($scfg);
+ return $res->{msg};
+};
+
+my $import_lun = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+
+ print "Import LUN\n";
+ return $create_lun->($scfg, $timeout, $method, @params);
+};
+
+my $modify_lun = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+ my $lun;
+ my $res = {msg => undef};
+
+ my $path = $params[1];
+
+ foreach my $cfg (@{$SETTINGS->{luns}}) {
+ if ($cfg->{Path} eq $path) {
+ $lun = $cfg;
+ last;
+ }
+ }
+
+ my $tid = $get_target_tid->($scfg);
+ my $device = 'diskt'.$tid.'l'.$lun->{lun};
+
+
+ @params = ('-resync_dev', $device, '-handler', $HANDLER,'-noprompt');
+ $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params);
+ die $res->{msg} unless $res->{result};
+
+ $update_config->($scfg);
+ return $res->{msg};
+};
+
+my $add_view = sub {
+ my ($scfg, $timeout, $method, @params) = @_;
+ return '';
+};
+
+my $get_lun_cmd_map = sub {
+ my ($method) = @_;
+
+ my $cmdmap = {
+ create_lu => { cmd => $create_lun },
+ delete_lu => { cmd => $delete_lun },
+ import_lu => { cmd => $import_lun },
+ modify_lu => { cmd => $modify_lun },
+ add_view => { cmd => $add_view },
+ list_view => { cmd => $list_view },
+ list_lu => { cmd => $list_lun },
+ };
+
+ die "unknown command '$method'" unless exists $cmdmap->{$method};
+
+ return $cmdmap->{$method};
+};
+
+sub run_lun_command {
+ my ($scfg, $timeout, $method, @params) = @_;
+
+ $readConfigFile->($scfg) unless $SETTINGS;
+ my $cmdmap = $get_lun_cmd_map->($method);
+ my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params);
+
+ return $msg;
+}
+
+sub get_base {
+ return '/dev/zvol';
+}
+
+1;
+
diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
index f88fe94..6694d4e 100644
--- a/PVE/Storage/ZFSPlugin.pm
+++ b/PVE/Storage/ZFSPlugin.pm
@@ -12,9 +12,11 @@ use base qw(PVE::Storage::ZFSPoolPlugin);
use PVE::Storage::LunCmd::Comstar;
use PVE::Storage::LunCmd::Istgt;
use PVE::Storage::LunCmd::Iet;
+use PVE::Storage::LunCmd::Scst;
+use Data::Dumper;
-my @ssh_opts = ('-o', 'BatchMode=yes');
+my @ssh_opts = ('-o', 'BatchMode=yes','-o','PreferredAuthentications=publickey');
my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
my $id_rsa_path = '/etc/pve/priv/zfs';
@@ -31,7 +33,7 @@ my $lun_cmds = {
my $zfs_unknown_scsi_provider = sub {
my ($provider) = @_;
- die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]";
+ die "$provider: unknown iscsi provider. Available [comstar, istgt, iet, scst]";
};
my $zfs_get_base = sub {
@@ -43,6 +45,8 @@ my $zfs_get_base = sub {
return PVE::Storage::LunCmd::Istgt::get_base;
} elsif ($scfg->{iscsiprovider} eq 'iet') {
return PVE::Storage::LunCmd::Iet::get_base;
+ } elsif ($scfg->{iscsiprovider} eq 'scst') {
+ return PVE::Storage::LunCmd::Scst::get_base;
} else {
$zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
}
@@ -63,6 +67,8 @@ sub zfs_request {
$msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params);
} elsif ($scfg->{iscsiprovider} eq 'iet') {
$msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params);
+ } elsif ($scfg->{iscsiprovider} eq 'scst') {
+ $msg = PVE::Storage::LunCmd::Scst::run_lun_command($scfg, $timeout, $method, @params);
} else {
$zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
}
@@ -98,11 +104,13 @@ sub zfs_get_lu_name {
my $object = ($zvol =~ /^.+\/.+/) ? "$base/$zvol" : "$base/$scfg->{pool}/$zvol";
+ print "$object\n";
+
my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object);
return $lu_name if $lu_name;
- die "Could not find lu_name for zvol $zvol";
+ die "Could not find lu_name for zvol $zvol: $object";
}
sub zfs_add_lun_mapping_entry {
@@ -351,6 +359,7 @@ sub volume_has_feature {
clone => { base => 1},
template => { current => 1},
copy => { base => 1, current => 1},
+ sparseinit => { base => 1, current => 1},
};
my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) =
--
2.11.0
More information about the pve-devel
mailing list