[pve-devel] [PATCH storage v3 2/5] Cephfs storage plugin

Thomas Lamprecht t.lamprecht at proxmox.com
Tue Jun 12 14:48:48 CEST 2018


On 6/11/18 11:31 AM, Alwin Antreich wrote:
>  - ability to mount through kernel and fuse client
>  - allow mount options
>  - get MONs from ceph config if not in storage.cfg
>  - allow the use of ceph config with fuse client
> 
> Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
> ---
>  PVE/API2/Storage/Config.pm  |   2 +-
>  PVE/Storage.pm              |   2 +
>  PVE/Storage/CephFSPlugin.pm | 225 ++++++++++++++++++++++++++++++++++++++++++++
>  PVE/Storage/Makefile        |   2 +-
>  PVE/Storage/Plugin.pm       |   1 +
>  debian/control              |   1 +
>  6 files changed, 231 insertions(+), 2 deletions(-)
>  create mode 100644 PVE/Storage/CephFSPlugin.pm
> 
> diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm
> index 3b38304..368a5c9 100755
> --- a/PVE/API2/Storage/Config.pm
> +++ b/PVE/API2/Storage/Config.pm
> @@ -171,7 +171,7 @@ __PACKAGE__->register_method ({
>  		    PVE::Storage::activate_storage($cfg, $baseid);
>  
>  		    PVE::Storage::LVMPlugin::lvm_create_volume_group($path, $opts->{vgname}, $opts->{shared});
> -		} elsif ($type eq 'rbd' && !defined($opts->{monhost})) {
> +		} elsif (($type eq 'rbd' || $type eq 'cephfs') && !defined($opts->{monhost})) {
>  		    my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring';
>  		    my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring";
>  
> diff --git a/PVE/Storage.pm b/PVE/Storage.pm
> index d733380..f9732fe 100755
> --- a/PVE/Storage.pm
> +++ b/PVE/Storage.pm
> @@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin;
>  use PVE::Storage::CIFSPlugin;
>  use PVE::Storage::ISCSIPlugin;
>  use PVE::Storage::RBDPlugin;
> +use PVE::Storage::CephFSPlugin;
>  use PVE::Storage::SheepdogPlugin;
>  use PVE::Storage::ISCSIDirectPlugin;
>  use PVE::Storage::GlusterfsPlugin;
> @@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register();
>  PVE::Storage::CIFSPlugin->register();
>  PVE::Storage::ISCSIPlugin->register();
>  PVE::Storage::RBDPlugin->register();
> +PVE::Storage::CephFSPlugin->register();
>  PVE::Storage::SheepdogPlugin->register();
>  PVE::Storage::ISCSIDirectPlugin->register();
>  PVE::Storage::GlusterfsPlugin->register();
> diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm
> new file mode 100644
> index 0000000..9882760
> --- /dev/null
> +++ b/PVE/Storage/CephFSPlugin.pm
> @@ -0,0 +1,225 @@
> +package PVE::Storage::CephFSPlugin;
> +
> +use strict;
> +use warnings;
> +use IO::File;
> +use Net::IP;
> +use File::Path;
> +use PVE::Tools qw(run_command);
> +use PVE::ProcFSTools;
> +use PVE::Storage::Plugin;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::CephStorageTools;

You add this module in the next patch (3/5) but use it already here,
which breaks build! Swap 3/5 before 2/5..
I'd also call it PVE::Storage::CephTools and put it there if we
have this here.

> +
> +use base qw(PVE::Storage::Plugin);
> +
> +my $parse_ceph_config = sub {

AFAIS, this is the direct copy of PVE::CephTools::parse_ceph_config ?
Why not move this into PVE::Storage::CephTools (called
PVE::CephStorageTools here) together with the monaddr function and
potential other functions from the existing package PVE::CephTools,
ideally making it obsolete so that only one is left.

> +    my ($filename) = @_;
> +
> +    my $cfg = {};
> +
> +    return $cfg if ! -f $filename;
> +
> +    my $fh = IO::File->new($filename, "r") ||
> +	die "unable to open '$filename' - $!\n";
> +
> +    my $section;
> +
> +    while (defined(my $line = <$fh>)) {
> +	$line =~ s/[;#].*$//;
> +	$line =~ s/^\s+//;
> +	$line =~ s/\s+$//;
> +	next if !$line;
> +
> +	$section = $1 if $line =~ m/^\[(\S+)\]$/;
> +	if (!$section) {
> +	    warn "no section - skip: $line\n";
> +	    next;
> +	}
> +
> +	if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
> +	    $cfg->{$section}->{$1} = $2;
> +	}
> +
> +    }
> +
> +    return $cfg;
> +};
> +
> +my $get_monaddr_list = sub {
> +    my ($scfg, $configfile) = @_;
> +
> +    my $server;
> +    my $no_mon = !defined($scfg->{monhost});
> +
> +    if (($no_mon) && defined($configfile)) {
> +	my $config = $parse_ceph_config->($configfile);
> +	$server = join(',', sort { $a cmp $b }

$a cmp $b is default, so not needed?

> +	    map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config});
> +    }else {

if $no_mon is false and $configfile is also we still get here,
with an undefined $scfg->{monhost}, can this happen? If so, I'd
error out above

> +	$server = PVE::CephStorageTools::hostlist($scfg->{monhost}, ',');
> +    }
> +
> +    return $server;
> +};
> +
> +sub cephfs_is_mounted {
> +    my ($scfg, $storeid, $mountdata) = @_;
> +
> +    my $cmd_option = PVE::CephStorageTools::ceph_connect_option($scfg, $storeid);
> +    my $configfile = $cmd_option->{ceph_conf};
> +    my $server = $get_monaddr_list->($scfg, $configfile);
> +
> +    my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/';

We normally use the following style for this:

my $subdir = $scfg->{subdir} // '/';

> +    my $mountpoint = $scfg->{path};
> +    my $source = "$server:$subdir";
> +
> +    $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
> +    return $mountpoint if grep {
> +	$_->[2] =~ m#^ceph|fuse\.ceph-fuse# &&
> +	$_->[0] =~ m#^\Q$source\E|ceph-fuse$# &&
> +	$_->[1] eq $mountpoint
> +    } @$mountdata;
> +
> +    warn "A filesystem is already mounted on $mountpoint\n"
> +	if grep { $_->[1] eq $mountpoint } @$mountdata;
> +
> +    return undef;
> +}
> +
> +sub cephfs_mount {
> +    my ($scfg, $storeid) = @_;
> +
> +    my $cmd;
> +    my $mountpoint = $scfg->{path};
> +    my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/';
> +
> +    my $cmd_option = PVE::CephStorageTools::ceph_connect_option($scfg, $storeid);
> +    my $configfile = $cmd_option->{ceph_conf};
> +    my $secretfile = $cmd_option->{keyring};
> +    my $server = $get_monaddr_list->($scfg, $configfile);
> +
> +    # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg
> +    # kernel -> better performance, less frequent updates
> +    if ($scfg->{fuse}) {
> +	    # FIXME: ceph-fuse client complains about missing ceph.conf or keyring if
> +	    # not provided on its default locations but still connects. Fix upstream??
> +	    $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$cmd_option->{userid}", '-m', $server];
> +	    push @$cmd, '--keyfile', $secretfile if defined($secretfile);
> +	    push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|);
> +	    push @$cmd, $mountpoint;
> +	    push @$cmd, '--conf', $configfile if defined($configfile);
> +    }else {
> +	my $source = "$server:$subdir";
> +	$cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$cmd_option->{userid}"];
> +	push @$cmd, '-o', "secretfile=$secretfile" if defined($secretfile);
> +    }
> +
> +    if ($scfg->{options}) {
> +	push @$cmd, '-o', $scfg->{options};
> +    }
> +
> +    run_command($cmd, errmsg => "mount error");
> +}
> +
> +# Configuration
> +
> +sub type {
> +    return 'cephfs';
> +}
> +
> +sub plugindata {
> +    return {
> +	content => [ { vztmpl => 1, iso => 1, backup => 1},
> +		     { backup => 1 }],
> +    };
> +}
> +
> +sub properties {
> +    return {
> +	fuse => {
> +	    description => "Mount CephFS through FUSE.",
> +	    type => 'boolean',
> +	},
> +	subdir => {
> +	    description => "Subdir to mount.",
> +	    type => 'string', format => 'pve-storage-path',
> +	},
> +    };
> +}
> +
> +sub options {
> +    return {
> +	path => { fixed => 1 },
> +	monhost => { optional => 1},
> +	nodes => { optional => 1 },
> +	subdir => { optional => 1 },
> +	disable => { optional => 1 },
> +	options => { optional => 1 },
> +	username => { optional => 1 },

username? for what? gets nowhere used...
maybe a copy-is-my-hobby error from the CIFS plugin? :-)

> +	content => { optional => 1 },
> +	format => { optional => 1 },
> +	mkdir => { optional => 1 },
> +	fuse => { optional => 1 },
> +	bwlimit => { optional => 1 },
> +    };
> +}
> +
> +sub check_config {
> +    my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
> +
> +    $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path};
> +
> +    return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck);
> +}
> +
> +# Storage implementation
> +
> +sub status {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
> +	if !$cache->{mountdata};
> +
> +    return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata});
> +
> +    return $class->SUPER::status($storeid, $scfg, $cache);
> +}
> +
> +sub activate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
> +	if !$cache->{mountdata};
> +
> +    my $path = $scfg->{path};
> +
> +    if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
> +
> +	# NOTE: only call mkpath when not mounted (avoid hang
> +	# when cephfs is offline
> +
> +	mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir});

what? I do not understand this, who set's mkdir and why should it
tell us if cephfs is not mounted? 
I'd just omit above comment, it's more confusing than helpful, IMO.

> +
> +	die "unable to activate storage '$storeid' - " .
> +	    "directory '$path' does not exist\n" if ! -d $path;
> +
> +	cephfs_mount($scfg, $storeid);
> +    }
> +
> +    $class->SUPER::activate_storage($storeid, $scfg, $cache);
> +}
> +
> +sub deactivate_storage {
> +    my ($class, $storeid, $scfg, $cache) = @_;
> +
> +    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
> +	if !$cache->{mountdata};
> +
> +    my $path = $scfg->{path};
> +
> +    if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
> +	my $cmd = ['/bin/umount', $path];
> +	run_command($cmd, errmsg => 'umount error'); 

trailing whitespace

> +    }
> +}
> diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
> index 7b168fa..ad69532 100644
> --- a/PVE/Storage/Makefile
> +++ b/PVE/Storage/Makefile
> @@ -1,4 +1,4 @@
> -SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
> +SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
>  
>  .PHONY: install
>  install:
> diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
> index 163871d..3769e01 100644
> --- a/PVE/Storage/Plugin.pm
> +++ b/PVE/Storage/Plugin.pm
> @@ -24,6 +24,7 @@ our @SHARED_STORAGE = (
>      'nfs',
>      'cifs',
>      'rbd',
> +    'cephfs',
>      'sheepdog',
>      'iscsidirect',
>      'glusterfs',
> diff --git a/debian/control b/debian/control
> index 908dd24..9923350 100644
> --- a/debian/control
> +++ b/debian/control
> @@ -28,6 +28,7 @@ Depends: cstream,
>           udev,
>           smbclient,
>           ceph-common,
> +         ceph-fuse,
>           cifs-utils,
>           ${perl:Depends},
>  Description: Proxmox VE storage management library
> 





More information about the pve-devel mailing list