[pve-devel] [PATCH manager 1/2] Create/destroy Ceph MDS

Alwin Antreich a.antreich at proxmox.com
Wed Nov 14 13:23:51 CET 2018


* introduce generic ceph service create/destroy methods
* add API for createmds/destroymds
* add createmds/destroymds to pveceph CLI

Signed-off-by: Alwin Antreich <a.antreich at proxmox.com>
---
 PVE/API2/Makefile  |   1 +
 PVE/API2/Ceph.pm   |   6 +++
 PVE/API2/CephFS.pm | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 PVE/CLI/pveceph.pm |   3 ++
 PVE/CephTools.pm   |  73 ++++++++++++++++++++++++++++-
 5 files changed, 213 insertions(+), 1 deletion(-)
 create mode 100644 PVE/API2/CephFS.pm

diff --git a/PVE/API2/Makefile b/PVE/API2/Makefile
index 9862e498..29ffac01 100644
--- a/PVE/API2/Makefile
+++ b/PVE/API2/Makefile
@@ -4,6 +4,7 @@ PERLSOURCE = 			\
 	Replication.pm		\
 	ReplicationConfig.pm	\
 	Ceph.pm			\
+	CephFS.pm		\
 	APT.pm			\
 	Subscription.pm		\
 	VZDump.pm		\
diff --git a/PVE/API2/Ceph.pm b/PVE/API2/Ceph.pm
index 8584cb51..20525c78 100644
--- a/PVE/API2/Ceph.pm
+++ b/PVE/API2/Ceph.pm
@@ -575,6 +575,11 @@ __PACKAGE__->register_method ({
 });
 
 __PACKAGE__->register_method ({
+    subclass => "PVE::API2::CephFS",
+    path => 'fs',
+});
+
+__PACKAGE__->register_method ({
     name => 'index',
     path => '',
     method => 'GET',
@@ -605,6 +610,7 @@ __PACKAGE__->register_method ({
 	    { name => 'mon' },
 	    { name => 'osd' },
 	    { name => 'pools' },
+	    { name => 'fs' },
 	    { name => 'stop' },
 	    { name => 'start' },
 	    { name => 'status' },
diff --git a/PVE/API2/CephFS.pm b/PVE/API2/CephFS.pm
new file mode 100644
index 00000000..b1c32b4f
--- /dev/null
+++ b/PVE/API2/CephFS.pm
@@ -0,0 +1,131 @@
+package PVE::API2::CephFS;
+
+use strict;
+use warnings;
+use Cwd qw(abs_path);
+use PVE::CephTools;
+use PVE::RPCEnvironment;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::RADOS;
+use PVE::RESTHandler;
+
+use base qw(PVE::RESTHandler);
+
+my $create_mds = sub {
+    my ($rados, $id) = @_;
+
+    my $priv = [
+	mon => 'allow profile mds',
+	osd => 'allow rwx',
+	mds => 'allow *',
+    ];
+
+    # $id needs to be literally in the config
+    my $keyring = '/var/lib/ceph/mds/ceph-$id/keyring';
+
+    my $cfg = PVE::CephTools::parse_ceph_config();
+
+    if (!defined($cfg->{'mds'}->{'keyring'})) {
+	$cfg->{'mds'}->{'keyring'} = $keyring;
+	PVE::CephTools::write_ceph_config($cfg);
+    }
+
+    PVE::CephTools::create_ceph_service($rados, $id, 'mds', $priv);
+};
+my $destroy_mds = sub {
+    my ($rados, $mds_id) = @_;
+
+    PVE::CephTools::destroy_ceph_service($rados, $mds_id, 'mds');
+};
+
+__PACKAGE__->register_method ({
+    name => 'createmds',
+    path => 'mds',
+    method => 'POST',
+    description => "Create Ceph Metadata Server",
+    proxyto => 'node',
+    protected => 1,
+    permissions => {
+	check => ['perm', '/', [ 'Sys.Modify' ]],
+    },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    id => {
+		type => 'string',
+		optional => 1,
+		pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
+		description => "The ID for the mds, when omitted the same as the nodename",
+	    },
+	},
+    },
+    returns => { type => 'string' },
+    code => sub {
+	my ($param) = @_;
+
+	PVE::CephTools::check_ceph_installed('ceph_mds');
+
+	PVE::CephTools::check_ceph_inited();
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	my $mdsid = $param->{id} // $param->{node};
+
+	my $worker = sub  {
+	    my $upid = shift;
+
+	    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
+
+	    $create_mds->($rados, $mdsid);
+	};
+
+	return $rpcenv->fork_worker('cephcreatemds', "mds.$mdsid", $authuser, $worker);
+    }});
+
+__PACKAGE__->register_method ({
+    name => 'destroymds',
+    path => 'mds/{id}',
+    method => 'DELETE',
+    description => "Destroy Ceph Metadata Server",
+    proxyto => 'node',
+    protected => 1,
+    permissions => {
+	check => ['perm', '/', [ 'Sys.Modify' ]],
+    },
+    parameters => {
+	additionalProperties => 0,
+	properties => {
+	    node => get_standard_option('pve-node'),
+	    id => {
+		description => 'The ID of the mds',
+		type => 'string',
+		pattern => '[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?',
+	    },
+	},
+    },
+    returns => { type => 'string' },
+    code => sub {
+	my ($param) = @_;
+
+	my $rpcenv = PVE::RPCEnvironment::get();
+
+	my $authuser = $rpcenv->get_user();
+
+	PVE::CephTools::check_ceph_inited();
+
+	my $mdsid = $param->{id};
+
+	my $worker = sub {
+	    my $upid = shift;
+
+	    my $rados = PVE::RADOS->new(timeout => PVE::CephTools::get_config('long_rados_timeout'));
+
+	    $destroy_mds->($rados, $mdsid);
+	};
+
+	return $rpcenv->fork_worker('cephdestroymds', "mds.$mdsid",  $authuser, $worker);
+    }});
+
diff --git a/PVE/CLI/pveceph.pm b/PVE/CLI/pveceph.pm
index a5a04949..02ed565e 100755
--- a/PVE/CLI/pveceph.pm
+++ b/PVE/CLI/pveceph.pm
@@ -19,6 +19,7 @@ use PVE::Tools qw(run_command);
 use PVE::JSONSchema qw(get_standard_option);
 use PVE::CephTools;
 use PVE::API2::Ceph;
+use PVE::API2::CephFS;
 
 use PVE::CLIHandler;
 
@@ -175,6 +176,8 @@ our $cmddef = {
     destroymon => [ 'PVE::API2::Ceph', 'destroymon', ['monid'], { node => $nodename }, $upid_exit],
     createmgr => [ 'PVE::API2::Ceph', 'createmgr', [], { node => $nodename }, $upid_exit],
     destroymgr => [ 'PVE::API2::Ceph', 'destroymgr', ['id'], { node => $nodename }, $upid_exit],
+    createmds => [ 'PVE::API2::CephFS', 'createmds', [], { node => $nodename }, $upid_exit],
+    destroymds => [ 'PVE::API2::CephFS', 'destroymds', ['id'], { node => $nodename }, $upid_exit],
     start => [ 'PVE::API2::Ceph', 'start', ['service'], { node => $nodename }, $upid_exit],
     stop => [ 'PVE::API2::Ceph', 'stop', ['service'], { node => $nodename }, $upid_exit],
     install => [ __PACKAGE__, 'install', [] ],
diff --git a/PVE/CephTools.pm b/PVE/CephTools.pm
index b104b5e8..25160d73 100644
--- a/PVE/CephTools.pm
+++ b/PVE/CephTools.pm
@@ -24,7 +24,8 @@ my $ceph_service = {
     ceph_bin => "/usr/bin/ceph",
     ceph_mon => "/usr/bin/ceph-mon",
     ceph_mgr => "/usr/bin/ceph-mgr",
-    ceph_osd => "/usr/bin/ceph-osd"
+    ceph_osd => "/usr/bin/ceph-osd",
+    ceph_mds => "/usr/bin/ceph-mds"
 };
 
 my $config_hash = {
@@ -228,4 +229,74 @@ sub systemd_managed {
     }
 }
 
+sub create_ceph_service {
+    my ($rados, $id, $service, $priv) = @_;
+
+    # eg. ceph fs status fails with numeric only ID.
+    die "ID: $id, numeric only IDs are not supported\n"
+	if $id =~ /^\d+$/;
+
+    # TODO: add check if service already exists in the cluster with the same ID
+
+    my $cluster_name = PVE::CephTools::get_config('ccname');
+    my $service_dir = "/var/lib/ceph/$service/$cluster_name-$id";
+    my $service_keyring = "$service_dir/keyring";
+    my $service_name = "$service.$id";
+
+    die "ceph $service directory '$service_dir' already exists\n"
+	if -d $service_dir;
+
+    print "creating $service directory '$service_dir'\n";
+    eval { File::Path::mkpath($service_dir) };
+    my $err = $@;
+    die "creation $service directory '$service_dir' failed\n" if $err;
+
+    print "creating keys for '$service_name'\n";
+    my $output = $rados->mon_command({ prefix => 'auth get-or-create',
+				       entity => $service_name,
+				       caps => $priv,
+				       format => 'plain'});
+    PVE::Tools::file_set_contents($service_keyring, $output);
+
+    print "setting owner for directory\n";
+    run_command(["chown", 'ceph:ceph', '-R', $service_dir]);
+
+    print "enabling service 'ceph-$service\@$id.service'\n";
+    ceph_service_cmd('enable', $service_name);
+    print "starting service 'ceph-$service\@$id.service'\n";
+    ceph_service_cmd('start', $service_name);
+
+    return undef;
+};
+
+sub destroy_ceph_service {
+    my ($rados, $id, $service) = @_;
+
+    my $cluster_name = PVE::CephTools::get_config('ccname');
+    my $service_name = "$service.$id";
+    my $service_dir = "/var/lib/ceph/$service/$cluster_name-$id";
+
+    warn "ceph $service directory '$service_dir' not found\n"
+	if ! -d $service_dir;
+
+    print "disabling service 'ceph-$service\@$id.service'\n";
+    PVE::CephTools::ceph_service_cmd('disable', $service_name);
+    print "stopping service 'ceph-$service\@$id.service'\n";
+    PVE::CephTools::ceph_service_cmd('stop', $service_name);
+
+    if (-d $service_dir) {
+	print "removing ceph-$service directory '$service_dir'\n";
+	File::Path::remove_tree($service_dir);
+    }
+
+    print "removing ceph user $service_name\n";
+    $rados->mon_command({
+	    prefix => 'auth del',
+	    entity => $service_name,
+	    format => 'plain'
+	});
+
+    return undef;
+};
+
 1;
-- 
2.11.0





More information about the pve-devel mailing list