[pve-devel] [PATCH v4 storage 1/1] Linux LIO/targetcli support

Stoiko Ivanov s.ivanov at proxmox.com
Wed Aug 1 15:24:09 CEST 2018


Hi,

Both Dominik and I gave your code another spin - and it worked as
expected. 
Additionally we both looked over the code and it looks far better than
the last version.

We discussed a further refactoring of the LunCmd modules, but that's a
different task altogether.

Thank you for your time and contribution!

Tested-by: Stoiko Ivanov <s.ivanov at proxmox.com>


On Thu, 26 Jul 2018 23:26:27 +0200
Udo Rader <udo.rader at bestsolution.at> wrote:

> Introducing LIO/targetcli support allowing to use recent linux
> distributions as iSCSI targets for ZFS volumes.
> 
> In order for this to work, two preconditions have to be met:
> 
> 1. the portal has to be set up correctly using targetcli
> 2. the initiator has to be authorized to connect to the target
>    based on the initiator's InitiatorName
> 
> When adding a LIO iSCSI target, a new "LIO target portal group" field
> needs to be correctly populated in the "Add: ZFS over iSCSI" popup,
> containing the fitting "LIO target portal group" name (typically
> something like 'tpg1'). ---
>  PVE/Storage/LunCmd/LIO.pm   | 387
> ++++++++++++++++++++++++++++++++++++ PVE/Storage/LunCmd/Makefile |
> 2 +- PVE/Storage/ZFSPlugin.pm    |  12 +-
>  3 files changed, 399 insertions(+), 2 deletions(-)
>  create mode 100644 PVE/Storage/LunCmd/LIO.pm
> 
> diff --git a/PVE/Storage/LunCmd/LIO.pm b/PVE/Storage/LunCmd/LIO.pm
> new file mode 100644
> index 0000000..a2050d4
> --- /dev/null
> +++ b/PVE/Storage/LunCmd/LIO.pm
> @@ -0,0 +1,387 @@
> +package PVE::Storage::LunCmd::LIO;
> +
> +# lightly based on code from Iet.pm 
> +#
> +# additional changes:
> +# -----------------------------------------------------------------
> +# Copyright (c) 2018 BestSolution.at EDV Systemhaus GmbH
> +# All Rights Reserved.
> +#
> +# This software is released under the terms of the
> +#
> +#            "GNU Affero General Public License"
> +# 
> +# and may only be distributed and used under the terms of the
> +# mentioned license. You should have received a copy of the license
> +# along with this software product, if not you can download it from
> +# https://www.gnu.org/licenses/agpl-3.0.en.html
> +# 
> +# Author: udo.rader at bestsolution.at
> +# -----------------------------------------------------------------
> +
> +use strict;
> +use warnings;
> +use PVE::Tools qw(run_command);
> +use JSON;
> +
> +sub get_base;
> +
> +# targetcli constants
> +# config file location differs from distro to distro
> +my @CONFIG_FILES = (
> +	'/etc/rtslib-fb-target/saveconfig.json',	# Debian 9.x
> et al	
> +	'/etc/target/saveconfig.json' ,			#
> ArchLinux, CentOS +);
> +my $BACKSTORE = '/backstores/block';
> +
> +my $SETTINGS = undef;
> +my $SETTINGS_TIMESTAMP = 0;
> +my $SETTINGS_MAXAGE = 15; # in seconds
> +
> +my @ssh_opts = ('-o', 'BatchMode=yes');
> +my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts);
> +my $id_rsa_path = '/etc/pve/priv/zfs';
> +my $targetcli = '/usr/bin/targetcli';
> +
> +my $execute_remote_command = sub {
> +    my ($scfg, $timeout, $remote_command, @params) = @_;
> +
> +    my $msg = '';
> +    my $err = undef;
> +    my $target;
> +    my $cmd;
> +    my $res = ();
> +
> +    $timeout = 10 if !$timeout;
> +
> +    my $output = sub {
> +        my $line = shift;
> +        $msg .= "$line\n";
> +    };
> +
> +    my $errfunc = sub {
> +        my $line = shift;
> +        $err .= "$line";
> +    };
> +
> +    $target = 'root@' . $scfg->{portal};
> +    $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa",
> $target, '--', $remote_command, @params]; +
> +    eval {
> +        run_command($cmd, outfunc => $output, errfunc => $errfunc,
> timeout => $timeout);
> +    };
> +    if ($@) {
> +        $res = {
> +            result => 0,
> +            msg => $err,
> +        }
> +    } else {
> +        $res = {
> +            result => 1,
> +            msg => $msg,
> +        }
> +    }
> +
> +    return $res;
> +};
> +
> +# fetch targetcli configuration from the portal
> +my $read_config = sub {
> +    my ($scfg, $timeout) = @_;
> +
> +    my $msg = '';
> +    my $err = undef;
> +    my $luncmd = 'cat';
> +    my $target;
> +    my $retry = 1;
> +
> +    $timeout = 10 if !$timeout;
> +
> +    my $output = sub {
> +        my $line = shift;
> +        $msg .= "$line\n";
> +    };
> +
> +    my $errfunc = sub {
> +        my $line = shift;
> +        $err .= "$line";
> +    };
> +
> +    $target = 'root@' . $scfg->{portal};
> +
> +    foreach my $oneFile (@CONFIG_FILES) {
> +        my $cmd = [@ssh_cmd, '-i',
> "$id_rsa_path/$scfg->{portal}_id_rsa", $target, $luncmd, $oneFile];
> +        eval {
> +            run_command($cmd, outfunc => $output, errfunc =>
> $errfunc, timeout => $timeout);
> +        };
> +        if ($@) {
> +            die $err if ($err !~ /No such file or directory/);
> +        }
> +        return $msg if $msg ne '';
> +    }
> +
> +    die "No configuration found. Install targetcli on
> $scfg->{portal}\n" if $msg eq ''; +
> +    return $msg;
> +};
> +
> +my $get_config = sub {
> +    my ($scfg) = @_;
> +    my @conf = undef;
> +
> +    my $config = $read_config->($scfg, undef);
> +    die "Missing config file" unless $config;
> +
> +    return $config;
> +};
> +
> +# fetches and parses targetcli config from the portal
> +my $parser = sub {
> +    my ($scfg) = @_;
> +    my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set,
> aborting!\n";
> +    my $tpg_tag;
> +
> +    if ($tpg =~ /^tpg(\d+)$/) {
> +	$tpg_tag = $1;
> +    } else {
> +	die "Target Portal Group has invalid value, must contain
> string 'tpg' and a suffix number, eg 'tpg17'\n";
> +    }
> +
> +    my $base = get_base;
> +
> +    my $config = $get_config->($scfg);
> +    my $jsonconfig = JSON->new->utf8->decode($config);
> +
> +    my $haveTarget = 0;
> +    foreach my $oneTarget (@{$jsonconfig->{targets}}) {
> +	# only interested in iSCSI targets
> +        if ($oneTarget->{fabric} eq 'iscsi' && $oneTarget->{wwn} eq
> $scfg->{target}) {
> +	    # find correct TPG
> +	    foreach my $oneTpg (@{$oneTarget->{tpgs}}) {
> +                if ($oneTpg->{tag} == $tpg_tag) {
> +                    $SETTINGS->{target} = $oneTpg;
> +                    $haveTarget = 1;
> +                    last;
> +		}
> +	    }
> +	}
> +    }
> +
> +    # seriously unhappy if the target server lacks iSCSI target
> configuration ...
> +    if (!$haveTarget) {
> +	die "target portal group tpg$tpg_tag not found!\n";
> +    }
> +};
> +
> +# removes the given lu_name from the local list of luns
> +my $free_lu_name = sub {
> +    my ($lu_name) = @_;
> +    my $new;
> +
> +    foreach my $lun (@{$SETTINGS->{target}->{luns}}) {
> +        if ($lun->{storage_object} ne "$BACKSTORE/$lu_name") {
> +            push @$new, $lun;
> +        }
> +    }
> +
> +    $SETTINGS->{target}->{luns} = $new;
> +};
> +
> +# locally registers a new lun 
> +my $register_lun = sub {
> +    my ($scfg, $idx, $volname) = @_;
> +
> +    my $conf = {
> +        index => $idx,
> +        storage_object => "$BACKSTORE/$volname",
> +        is_new => 1,
> +    };
> +    push @{$SETTINGS->{target}->{luns}}, $conf;
> +
> +    return $conf;
> +};
> +
> +# extracts the ZFS volume name from a device path
> +my $extract_volname = sub {
> +    my ($scfg, $lunpath) = @_;
> +    my $volname = undef;
> +
> +    my $base = get_base;
> +    if ($lunpath =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) {
> +	$volname = $1;
> +    }
> +
> +    return $volname;
> +};
> +
> +# retrieves the LUN index for a particular object
> +my $list_view = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +    my $lun = undef;
> +
> +    my $object = $params[0];
> +    my $volname = $extract_volname->($scfg, $params[0]);
> +
> +    foreach my $lun (@{$SETTINGS->{target}->{luns}}) {
> +        if ($lun->{storage_object} eq "$BACKSTORE/$volname") {
> +            return $lun->{index};
> +        }
> +    }
> +
> +    return $lun;
> +};
> +
> +# determines, if the given object exists on the portal
> +my $list_lun = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +    my $name = undef;
> +
> +    my $object = $params[0];
> +    my $volname = $extract_volname->($scfg, $params[0]);
> +
> +    foreach my $lun (@{$SETTINGS->{target}->{luns}}) {
> +        if ($lun->{storage_object} eq "$BACKSTORE/$volname") {
> +            return $object;
> +        }
> +    }
> +
> +    return $name;
> +};
> +
> +# adds a new LUN to the target
> +my $create_lun = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +
> +    if ($list_lun->($scfg, $timeout, $method, @params)) {
> +        die "$params[0]: LUN already exists!";
> +    }
> +
> +    my $device = $params[0];
> +    my $volname = $extract_volname->($scfg, $device);
> +    my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set,
> aborting!\n"; +
> +    # step 1: create backstore for device
> +    my @cliparams = ($BACKSTORE, 'create', "name=$volname",
> "dev=$device" );
> +    my $res = $execute_remote_command->($scfg, $timeout, $targetcli,
> @cliparams);
> +    die $res->{msg} if !$res->{result};
> +
> +    # step 2: register lun with target
> +    #
> targetcli /iscsi/iqn.2018-04.at.bestsolution.somehost:target/tpg1/luns/
> create /backstores/block/foobar
> +    @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/", 'create',
> "$BACKSTORE/$volname" );
> +    $res = $execute_remote_command->($scfg, $timeout, $targetcli,
> @cliparams);
> +    die $res->{msg} if !$res->{result};
> +
> +    # targetcli responds with "Created LUN 99"
> +    # not calculating the index ourselves, because the index at the
> portal might have
> +    # changed without our knowledge, so relying on the number that
> targetcli returns
> +    my $lun_idx;
> +    if ($res->{msg} =~ /LUN (\d+)/) {
> +        $lun_idx = $1;
> +    } else {
> +        die "unable to determine new LUN index: $res->{msg}";
> +    }
> +
> +    $register_lun->($scfg, $lun_idx, $volname);
> +
> +    # step 3: unfortunately, targetcli doesn't always save changes,
> no matter 
> +    #         if auto_save_on_exit is true or not. So saving to be
> safe ...
> +    $execute_remote_command->($scfg, $timeout, $targetcli,
> 'saveconfig'); +
> +    return $res->{msg};
> +};
> +
> +my $delete_lun = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +    my $res = {msg => undef};
> +
> +    my $tpg = $scfg->{lio_tpg} || die "Target Portal Group not set,
> aborting!\n"; +
> +    my $path = $params[0];
> +    my $volname = $extract_volname->($scfg, $params[0]);
> +
> +    foreach my $lun (@{$SETTINGS->{target}->{luns}}) {
> +        if ($lun->{storage_object} eq "$BACKSTORE/$volname") {
> +            # step 1: delete the lun
> +            my @cliparams = ("/iscsi/$scfg->{target}/$tpg/luns/",
> 'delete', "lun$lun->{index}" );
> +            my $res = $execute_remote_command->($scfg, $timeout,
> $targetcli, @cliparams);
> +            do {
> +                die $res->{msg};
> +            } unless $res->{result};
> +
> +            # step 2: delete the backstore
> +            @cliparams = ($BACKSTORE, 'delete', $volname);
> +            $res = $execute_remote_command->($scfg, $timeout,
> $targetcli, @cliparams);
> +            do {
> +                die $res->{msg};
> +            } unless $res->{result};
> +
> +            # step 3: save to be safe ...
> +            $execute_remote_command->($scfg, $timeout, $targetcli,
> 'saveconfig'); +
> +            # update interal cache
> +            $free_lu_name->($volname);
> +
> +            last;
> +        }
> +    }
> +
> +    return $res->{msg};
> +};
> +
> +my $import_lun = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +
> +    return $create_lun->($scfg, $timeout, $method, @params);
> +};
> +
> +# needed for example when the underlying ZFS volume has been resized
> +my $modify_lun = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +    my $msg;
> +
> +    $msg = $delete_lun->($scfg, $timeout, $method, @params);
> +    if ($msg) {
> +        $msg = $create_lun->($scfg, $timeout, $method, @params);
> +    }
> +
> +    return $msg;
> +};
> +
> +my $add_view = sub {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +
> +    return '';
> +};
> +
> +my %lun_cmd_map = (
> +    create_lu   =>  $create_lun,
> +    delete_lu   =>  $delete_lun,
> +    import_lu   =>  $import_lun,
> +    modify_lu   =>  $modify_lun,
> +    add_view    =>  $add_view,
> +    list_view   =>  $list_view,
> +    list_lu     =>  $list_lun,
> +);
> +
> +sub run_lun_command {
> +    my ($scfg, $timeout, $method, @params) = @_;
> +
> +    # fetch configuration from target if we haven't yet
> +    # or if our configuration is stale
> +    my $timediff = time - $SETTINGS_TIMESTAMP;
> +    if ( ! $SETTINGS || $timediff > $SETTINGS_MAXAGE ) {
> +        $SETTINGS_TIMESTAMP = time;
> +        $parser->($scfg);
> +    }
> +
> +    die "unknown command '$method'" unless exists
> $lun_cmd_map{$method};
> +    my $msg = $lun_cmd_map{$method}->($scfg, $timeout, $method,
> @params); +
> +    return $msg;
> +}
> +
> +sub get_base {
> +    return '/dev';
> +}
> +
> +1;
> diff --git a/PVE/Storage/LunCmd/Makefile b/PVE/Storage/LunCmd/Makefile
> index b959255..a7209d1 100644
> --- a/PVE/Storage/LunCmd/Makefile
> +++ b/PVE/Storage/LunCmd/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES=Comstar.pm Istgt.pm Iet.pm
> +SOURCES=Comstar.pm Istgt.pm Iet.pm LIO.pm
>  
>  .PHONY: install
>  install:
> diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm
> index f88fe94..1fac811 100644
> --- a/PVE/Storage/ZFSPlugin.pm
> +++ b/PVE/Storage/ZFSPlugin.pm
> @@ -12,6 +12,7 @@ use base qw(PVE::Storage::ZFSPoolPlugin);
>  use PVE::Storage::LunCmd::Comstar;
>  use PVE::Storage::LunCmd::Istgt;
>  use PVE::Storage::LunCmd::Iet;
> +use PVE::Storage::LunCmd::LIO;
>  
>  
>  my @ssh_opts = ('-o', 'BatchMode=yes');
> @@ -31,7 +32,7 @@ my $lun_cmds = {
>  my $zfs_unknown_scsi_provider = sub {
>      my ($provider) = @_;
>  
> -    die "$provider: unknown iscsi provider. Available [comstar,
> istgt, iet]";
> +    die "$provider: unknown iscsi provider. Available [comstar,
> istgt, iet, LIO]"; };
>  
>  my $zfs_get_base = sub {
> @@ -43,6 +44,8 @@ my $zfs_get_base = sub {
>          return PVE::Storage::LunCmd::Istgt::get_base;
>      } elsif ($scfg->{iscsiprovider} eq 'iet') {
>          return PVE::Storage::LunCmd::Iet::get_base;
> +    } elsif ($scfg->{iscsiprovider} eq 'LIO') {
> +        return PVE::Storage::LunCmd::LIO::get_base;
>      } else {
>          $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
>      }
> @@ -63,6 +66,8 @@ sub zfs_request {
>              $msg =
> PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout,
> $method, @params); } elsif ($scfg->{iscsiprovider} eq 'iet') { $msg =
> PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method,
> @params);
> +        } elsif ($scfg->{iscsiprovider} eq 'LIO') {
> +            $msg = PVE::Storage::LunCmd::LIO::run_lun_command($scfg,
> $timeout, $method, @params); } else {
>              $zfs_unknown_scsi_provider->($scfg->{iscsiprovider});
>          }
> @@ -188,6 +193,10 @@ sub properties {
>  	    description => "host group for comstar views",
>  	    type => 'string',
>  	},
> +	lio_tpg => {
> +	    description => "target portal group for Linux LIO
> targets",
> +	    type => 'string',
> +	},
>      };
>  }
>  
> @@ -204,6 +213,7 @@ sub options {
>  	sparse => { optional => 1 },
>  	comstar_hg => { optional => 1 },
>  	comstar_tg => { optional => 1 },
> +	lio_tpg => { optional => 1 },
>  	content => { optional => 1 },
>  	bwlimit => { optional => 1 },
>      };





More information about the pve-devel mailing list