[pve-devel] [PATCH manager 6/8] ceph: add MDS create/delete/list API
Fabian Grünbichler
f.gruenbichler at proxmox.com
Wed Nov 21 16:07:35 CET 2018
LGTM in general, small nits inlines, further testing with GUI
integration expected.
On Wed, Nov 21, 2018 at 11:42:41AM +0100, Thomas Lamprecht wrote:
> Allow to create, list and destroy and Ceph Metadata Server (MDS) over
> the API and the CLI `pveceph` tool.
>
> Besides setting up the local systemd service template and the MDS
> data directory we also add a reference to the MDS in the ceph.conf
> We note the backing host (node) from the respective MDS and set up a
> 'mds standby for name' = 'pve' so that the PVE created ones are a
> single group. If we decide to add integration for rank/path specific
> MDS (possible useful for CephFS with quite a bit of load) then this
> may help as a starting point.
>
> On create, check early if a reference already exists in ceph.conf and
> abort in that case. If we only see existing data directories later
> on we do not remove them, they could well be from an older manual
> create - where it's possible dangerous to just remove it. Let the
> user handle it themself in that case.
>
> Signed-off-by: Thomas Lamprecht <t.lamprecht at proxmox.com>
> Co-authored-by: Alwin Antreich <a.antreich at proxmox.com>
> ---
>
> Not fully sure about the 'mds standby for name' so if you have objections
> regarding (negative) implications of this we can remove it just fine.
>
> PVE/API2/Ceph.pm | 7 ++
> PVE/API2/Ceph/MDS.pm | 196 +++++++++++++++++++++++++++++++++++++++++
> PVE/API2/Ceph/Makefile | 15 ++++
> PVE/API2/Makefile | 4 +
> PVE/CLI/pveceph.pm | 3 +
> PVE/CephTools.pm | 99 ++++++++++++++++++++-
> 6 files changed, 323 insertions(+), 1 deletion(-)
> create mode 100644 PVE/API2/Ceph/MDS.pm
> create mode 100644 PVE/API2/Ceph/Makefile
>
> diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
> index 854527a3..a8665dc1 100644
> --- a/PVE/API2/Ceph.pm
> +++ b/PVE/API2/Ceph.pm
> @@ -542,6 +542,7 @@ use PVE::Storage;
> use PVE::RESTHandler;
> use PVE::RPCEnvironment;
> use PVE::JSONSchema qw(get_standard_option);
> +use PVE::API2::Ceph::MDS;
> use PVE::RADOS;
> use PVE::CephTools;
> use PVE::Network;
> @@ -555,6 +556,11 @@ __PACKAGE__->register_method ({
> path => 'osd',
> });
>
> +__PACKAGE__->register_method ({
> + subclass => "PVE::API2::Ceph::MDS",
> + path => 'mds',
> +});
> +
> __PACKAGE__->register_method ({
> name => 'index',
> path => '',
> @@ -586,6 +592,7 @@ __PACKAGE__->register_method ({
> { name => 'mon' },
> { name => 'osd' },
> { name => 'pools' },
> + { name => 'mds' },
> { name => 'stop' },
> { name => 'start' },
> { name => 'status' },
> diff --git a/PVE/API2/Ceph/MDS.pm b/PVE/API2/Ceph/MDS.pm
> new file mode 100644
> index 00000000..990fd0fc
> --- /dev/null
> +++ b/PVE/API2/Ceph/MDS.pm
> @@ -0,0 +1,196 @@
> +package PVE::API2::Ceph::MDS;
> +
> +use strict;
> +use warnings;
> +
> +use PVE::CephTools;
> +use PVE::API2::Ceph;
> +use PVE::API2::Storage::Config;
> +use PVE::RPCEnvironment;
> +use PVE::JSONSchema qw(get_standard_option);
> +use PVE::RADOS;
> +use PVE::RESTHandler;
> +use PVE::INotify;
> +
> +use base qw(PVE::RESTHandler);
> +
> +__PACKAGE__->register_method ({
> + name => 'index',
> + path => '',
> + method => 'GET',
> + description => "Directory index.",
MDS index?
> + permissions => {
> + check => ['perm', '/', [ 'Sys.Audit', 'Datastore.Audit' ], any => 1],
> + },
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + },
> + },
> + returns => {
> + type => 'array',
> + items => {
> + type => "object",
> + properties => {},
> + },
> + links => [ { rel => 'child', href => "{id}" } ],
> + },
> + code => sub {
> + my ($param) = @_;
> +
> + my $mds_list = PVE::CephTools::list_local_mds_ids();
> +
> + my $res = [];
> + foreach my $mds (@$mds_list) {
> + push @$res, { 'id' => $mds };
> + }
> +
> + return $res;
> + }
> +});
> +
> +__PACKAGE__->register_method ({
> + name => 'createmds',
> + path => '{id}',
> + method => 'POST',
> + description => "Create Ceph Metadata Server (MDS)",
> + proxyto => 'node',
> + protected => 1,
> + permissions => {
> + check => ['perm', '/', [ 'Sys.Modify' ]],
> + },
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + id => {
> + type => 'string',
> + optional => 1,
> + default => 'nodename',
> + pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
> + description => "The ID for the mds, when omitted the same as the nodename",
s/mds/MDS
is the 'when omitted ...' necessary, since we already specify the
default?
> + },
> + hotstandby => {
> + type => 'boolean',
> + optional => 1,
> + default => '0',
> + description => "Determines whether a ceph-mds daemon should poll and replay the log of an active MDS. ".
> + "Faster switch on MDS failure, but needs more idle resources.",
> + },
> + },
> + },
> + returns => { type => 'string' },
> + code => sub {
> + my ($param) = @_;
> +
> + PVE::CephTools::check_ceph_installed('ceph_mds');
> +
> + PVE::CephTools::check_ceph_inited();
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> + my $authuser = $rpcenv->get_user();
> +
> + my $nodename = $param->{node};
> + $nodename = INotify::nodename() if $nodename eq 'localhost';
> +
> + my $mds_id = $param->{id} // $nodename;
> +
> + my $worker = sub {
> + my $timeout = PVE::CephTools::get_config('long_rados_timeout');
> + my $rados = PVE::RADOS->new(timeout => $timeout);
> +
> + my $cfg = PVE::CephTools::parse_ceph_config();
> +
> + my $section = "mds.$mds_id";
> +
> + if (defined($cfg->{$section})) {
> + die "MDS '$mds_id' already referenced in ceph config, abort!\n"
> + }
> +
> + if (!defined($cfg->{mds}->{keyring})) {
> + # $id isn't a perl variable but a ceph metavariable
> + my $keyring = '/var/lib/ceph/mds/ceph-$id/keyring';
> +
> + $cfg->{mds}->{keyring} = $keyring;
> + }
> +
> + $cfg->{$section}->{host} = $nodename;
> + $cfg->{$section}->{"mds standby for name"} = 'pve';
> +
> + if ($param->{hotstandby}) {
> + $cfg->{$section}->{"mds standby replay"} = 'true';
> + }
> +
> + PVE::CephTools::write_ceph_config($cfg);
> +
> + eval { PVE::CephTools::create_mds($mds_id, $rados) };
> + if (my $err = $@) {
> + # we abort early if the section is defined, so we know that we
> + # wrote it at this point. Do not auto remove the service, could
> + # do real harm for previously manual setup MDS
> + warn "Encountered error, remove '$section' from ceph.conf\n";
> + $cfg = PVE::CephTools::parse_ceph_config();
> + delete $cfg->{$section};
> + PVE::CephTools::write_ceph_config($cfg);
> +
> + die "$err\n";
> + }
> + };
> +
> + return $rpcenv->fork_worker('cephcreatemds', "mds.$mds_id", $authuser, $worker);
> + }
> +});
> +
> +__PACKAGE__->register_method ({
> + name => 'destroymds',
> + path => '{id}',
> + method => 'DELETE',
> + description => "Destroy Ceph Metadata Server",
> + proxyto => 'node',
> + protected => 1,
> + permissions => {
> + check => ['perm', '/', [ 'Sys.Modify' ]],
> + },
> + parameters => {
> + additionalProperties => 0,
> + properties => {
> + node => get_standard_option('pve-node'),
> + id => {
> + description => 'The ID of the mds',
> + type => 'string',
> + pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
> + },
> + },
> + },
> + returns => { type => 'string' },
> + code => sub {
> + my ($param) = @_;
> +
> + my $rpcenv = PVE::RPCEnvironment::get();
> +
> + my $authuser = $rpcenv->get_user();
> +
> + PVE::CephTools::check_ceph_inited();
> +
> + my $mds_id = $param->{id};
> +
> + my $worker = sub {
> + my $timeout = PVE::CephTools::get_config('long_rados_timeout');
> + my $rados = PVE::RADOS->new(timeout => $timeout);
> +
> + my $cfg = PVE::CephTools::parse_ceph_config();
> +
> + if (defined($cfg->{"mds.$mds_id"})) {
> + delete $cfg->{"mds.$mds_id"};
> + PVE::CephTools::write_ceph_config($cfg);
> + }
> +
> + PVE::CephTools::destroy_mds($mds_id, $rados);
> + };
> +
> + return $rpcenv->fork_worker('cephdestroymds', "mds.$mds_id", $authuser, $worker);
> + }
> +});
> +
> +1;
> diff --git a/PVE/API2/Ceph/Makefile b/PVE/API2/Ceph/Makefile
> new file mode 100644
> index 00000000..be4d740c
> --- /dev/null
> +++ b/PVE/API2/Ceph/Makefile
> @@ -0,0 +1,15 @@
> +include ../../../defines.mk
> +
> +PERLSOURCE= \
> + MDS.pm
> +
> +all:
> +
> +.PHONY: clean
> +clean:
> + rm -rf *~
> +
> +.PHONY: install
> +install: ${PERLSOURCE}
> + install -d ${PERLLIBDIR}/PVE/API2/Ceph
> + install -m 0644 ${PERLSOURCE} ${PERLLIBDIR}/PVE/API2/Ceph
> diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
> index 75b022dd..20649ea5 100644
> --- a/PVE/API2/Makefile
> +++ b/PVE/API2/Makefile
> @@ -1,5 +1,7 @@
> include ../../defines.mk
>
> +SUBDIRS=Ceph
> +
> PERLSOURCE = \
> Replication.pm \
> ReplicationConfig.pm \
> @@ -26,8 +28,10 @@ all:
> .PHONY: clean
> clean:
> rm -rf *~
> + set -e && for i in ${SUBDIRS}; do ${MAKE} -C $$i $@; done
>
> .PHONY: install
> install: ${PERLSOURCE}
> install -d ${PERLLIBDIR}/PVE/API2
> install -m 0644 ${PERLSOURCE} ${PERLLIBDIR}/PVE/API2
> + set -e && for i in ${SUBDIRS}; do ${MAKE} -C $$i $@; done
> diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
> index a5a04949..90878d9e 100755
> --- a/PVE/CLI/pveceph.pm
> +++ b/PVE/CLI/pveceph.pm
> @@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
> use PVE::JSONSchema qw(get_standard_option);
> use PVE::CephTools;
> use PVE::API2::Ceph;
> +use PVE::API2::Ceph::MDS;
>
> use PVE::CLIHandler;
>
> @@ -175,6 +176,8 @@ our $cmddef = {
> destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
> createmgr => [ 'PVE::API2::Ceph', 'createmgr', [], { node => $nodename }, $upid_exit],
> destroymgr => [ 'PVE::API2::Ceph', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
> + createmds => [ 'PVE::API2::Ceph::MDS', 'createmds', [], { node => $nodename }, $upid_exit],
> + destroymds => [ 'PVE::API2::Ceph::MDS', 'destroymds', ['id'], { node => $nodename }, $upid_exit],
> start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, $upid_exit],
> stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, $upid_exit],
> install => [ __PACKAGE__, 'install', [] ],
> diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
> index 93d531b6..d0e774ed 100644
> --- a/PVE/CephTools.pm
> +++ b/PVE/CephTools.pm
> @@ -17,12 +17,14 @@ my $pve_mon_key_path = "/etc/pve/priv/$ccname.mon.keyring";
> my $pve_ckeyring_path = "/etc/pve/priv/$ccname.client.admin.keyring";
> my $ceph_bootstrap_osd_keyring = "/var/lib/ceph/bootstrap-osd/$ccname.keyring";
> my $ceph_bootstrap_mds_keyring = "/var/lib/ceph/bootstrap-mds/$ccname.keyring";
> +my $ceph_mds_data_dir = '/var/lib/ceph/mds';
>
> my $ceph_service = {
> ceph_bin => "/usr/bin/ceph",
> ceph_mon => "/usr/bin/ceph-mon",
> ceph_mgr => "/usr/bin/ceph-mgr",
> - ceph_osd => "/usr/bin/ceph-osd"
> + ceph_osd => "/usr/bin/ceph-osd",
> + ceph_mds => "/usr/bin/ceph-mds",
> };
>
> my $config_hash = {
> @@ -32,6 +34,7 @@ my $config_hash = {
> pve_ckeyring_path => $pve_ckeyring_path,
> ceph_bootstrap_osd_keyring => $ceph_bootstrap_osd_keyring,
> ceph_bootstrap_mds_keyring => $ceph_bootstrap_mds_keyring,
> + ceph_mds_data_dir => $ceph_mds_data_dir,
> long_rados_timeout => 60,
> };
>
> @@ -296,4 +299,98 @@ sub systemd_managed {
> }
> }
>
> +sub list_local_mds_ids {
> + my $mds_list = [];
> +
> + PVE::Tools::dir_glob_foreach($ceph_mds_data_dir, qr/$ccname-(\S+)/, sub {
> + my (undef, $mds_id) = @_;
> + push @$mds_list, $mds_id;
> + });
> +
> + return $mds_list;
> +}
> +
> +sub create_mds {
> + my ($id, $rados) = @_;
> +
> + # `ceph fs status` fails with numeric only ID.
> + die "ID: $id, numeric only IDs are not supported\n"
> + if $id =~ /^\d+$/;
> +
> + if (!defined($rados)) {
> + $rados = PVE::RADOS->new();
> + }
> +
> + my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
> + my $service_keyring = "$service_dir/keyring";
> + my $service_name = "mds.$id";
> +
> + die "ceph MDS directory '$service_dir' already exists\n"
> + if -d $service_dir;
> +
> + print "creating MDS directory '$service_dir'\n";
> + eval { File::Path::mkpath($service_dir) };
> + my $err = $@;
> + die "creation MDS directory '$service_dir' failed\n" if $err;
s/creation/creation of
> +
> + # http://docs.ceph.com/docs/luminous/install/manual-deployment/#adding-mds
> + my $priv = [
> + mon => 'allow profile mds',
> + osd => 'allow rwx',
> + mds => 'allow *',
> + ];
> +
> + print "creating keys for '$service_name'\n";
> + my $output = $rados->mon_command({
> + prefix => 'auth get-or-create',
> + entity => $service_name,
> + caps => $priv,
> + format => 'plain',
> + });
> +
> + PVE::Tools::file_set_contents($service_keyring, $output);
> +
> + print "setting ceph as owner for service directory\n";
> + run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
> +
> + print "enabling service 'ceph-mds\@$id.service'\n";
> + ceph_service_cmd('enable', $service_name);
> + print "starting service 'ceph-mds\@$id.service'\n";
> + ceph_service_cmd('start', $service_name);
> +
> + return undef;
> +};
> +
> +sub destroy_mds {
> + my ($id, $rados) = @_;
> +
> + if (!defined($rados)) {
> + $rados = PVE::RADOS->new();
> + }
> +
> + my $service_name = "mds.$id";
> + my $service_dir = "/var/lib/ceph/mds/$ccname-$id";
> +
> + print "disabling service 'ceph-mds\@$id.service'\n";
> + ceph_service_cmd('disable', $service_name);
> + print "stopping service 'ceph-mds\@$id.service'\n";
> + ceph_service_cmd('stop', $service_name);
> +
> + if (-d $service_dir) {
> + print "removing ceph-mds directory '$service_dir'\n";
> + File::Path::remove_tree($service_dir);
> + } else {
> + warn "cannot cleanup MDS $id directory, '$service_dir' not found\n"
> + }
> +
> + print "removing ceph auth for '$service_name'\n";
> + $rados->mon_command({
> + prefix => 'auth del',
> + entity => $service_name,
> + format => 'plain'
> + });
> +
> + return undef;
> +};
> +
> 1;
> --
> 2.19.1
>
>
> _______________________________________________
> pve-devel mailing list
> pve-devel at pve.proxmox.com
> https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
More information about the pve-devel
mailing list