Full Mesh Network for Ceph Server

From Proxmox VE
Revision as of 11:40, 28 October 2016 by MathieuMD (talk | contribs) (→‎Introduction: Fix wikilink)
Jump to navigation Jump to search

Introduction

This wiki page describe a three node Ceph Server network configuration, using full mesh network (instead of a network switch). This should also work with a 5-node cluster, general you need nodes_total - 1 = nic ports. The basic idea is running a small 3 node cluster with 10 Gbit network WITHOUT buying an expensive 10 Gbit network switch.

This should work with 40 Gbit or even 100 Gbit Nics. But for this howto, 10 Gbit Intel NICs were used.

Node1

/etc/network/interface/

root@pve-2-50:~# cat /etc/network/interfaces

auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
       address  10.14.14.50
       netmask  255.255.255.0
auto eth0
iface eth0 inet static
       address  10.15.15.50
       netmask  255.255.255.0
       up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
       down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
auto eth1
iface eth1 inet static
       address  10.15.15.50
       netmask  255.255.255.0
       up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
       down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
       address  192.168.2.50
       netmask  255.255.240.0
       gateway  192.168.2.1
       bridge_ports eth2
       bridge_stp off
       bridge_fd 0

route

root@pve-2-50:~# route -n

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.51     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.52     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0

Node2

/etc/network/interface/

root@pve-2-51:/# cat /etc/network/interfaces

auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
       address  10.14.14.51
       netmask  255.255.255.0
auto eth0
iface eth0 inet static
       address  10.15.15.51
       netmask  255.255.255.0
       up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth0
       down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0
auto eth1
iface eth1 inet static
       address  10.15.15.51
       netmask  255.255.255.0
       up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
       down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
       address  192.168.2.51
       netmask  255.255.240.0
       gateway  192.168.2.1
       bridge_ports eth2
       bridge_stp off
       bridge_fd 0

route

root@pve-2-51:/# route -n

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.50     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.52     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0

Node3

/etc/network/interface/

root@pve-2-52:~# cat /etc/network/interfaces

auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
       address  10.14.14.52
       netmask  255.255.255.0
auto eth0
iface eth0 inet static
       address  10.15.15.52
       netmask  255.255.255.0
       up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
       down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
auto eth1
iface eth1 inet static
       address  10.15.15.52
       netmask  255.255.255.0
       up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth1
       down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
       address  192.168.2.52
       netmask  255.255.240.0
       gateway  192.168.2.1
       bridge_ports eth2
       bridge_stp off
       bridge_fd 0

route

root@pve-2-52:~# route -n

Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.51     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.50     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0