Difference between revisions of "Full Mesh Network for Ceph Server"
Jump to navigation
Jump to search
m (wikiformat + comments) |
|||
| Line 7: | Line 7: | ||
=Node1= | =Node1= | ||
==/etc/network/interface== | ==/etc/network/interface== | ||
| − | |||
<pre> | <pre> | ||
auto lo | auto lo | ||
| Line 19: | Line 18: | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| + | # Connected to Node2 (.51) | ||
auto eth0 | auto eth0 | ||
iface eth0 inet static | iface eth0 inet static | ||
| Line 26: | Line 26: | ||
down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0 | down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0 | ||
| + | # Connected to Node3 (.52) | ||
auto eth1 | auto eth1 | ||
iface eth1 inet static | iface eth1 inet static | ||
| Line 58: | Line 59: | ||
=Node2= | =Node2= | ||
| − | ==/etc/network/interface | + | ==/etc/network/interface== |
| + | <pre> | ||
| + | auto lo | ||
| + | iface lo inet loopback | ||
| − | + | iface eth2 inet manual | |
| − | + | auto eth3 | |
| − | + | iface eth3 inet static | |
| − | |||
| − | |||
| − | |||
| − | |||
| − | |||
address 10.14.14.51 | address 10.14.14.51 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| − | + | # Connected to Node1 (.50) | |
| − | + | auto eth0 | |
| + | iface eth0 inet static | ||
address 10.15.15.51 | address 10.15.15.51 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| Line 79: | Line 79: | ||
down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0 | down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0 | ||
| − | + | # Connected to Node3 (.52) | |
| − | + | auto eth1 | |
| + | iface eth1 inet static | ||
address 10.15.15.51 | address 10.15.15.51 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| Line 86: | Line 87: | ||
down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1 | down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1 | ||
| − | + | auto vmbr0 | |
| − | + | iface vmbr0 inet static | |
address 192.168.2.51 | address 192.168.2.51 | ||
netmask 255.255.240.0 | netmask 255.255.240.0 | ||
| Line 94: | Line 95: | ||
bridge_stp off | bridge_stp off | ||
bridge_fd 0 | bridge_fd 0 | ||
| + | </pre> | ||
==route== | ==route== | ||
| + | <pre> | ||
root@pve-2-51:/# route -n | root@pve-2-51:/# route -n | ||
| − | + | Kernel IP routing table | |
| − | + | Destination Gateway Genmask Flags Metric Ref Use Iface | |
| − | + | 10.15.15.50 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 | |
| − | + | 10.15.15.52 0.0.0.0 255.255.255.255 UH 0 0 0 eth1 | |
| − | + | 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 | |
| − | + | 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 | |
| − | + | 10.14.14.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3 | |
| − | + | 192.168.0.0 0.0.0.0 255.255.240.0 U 0 0 0 vmbr0 | |
| − | + | 0.0.0.0 192.168.2.1 0.0.0.0 UG 0 0 0 vmbr0 | |
| + | </pre> | ||
=Node3= | =Node3= | ||
| − | ==/etc/network/interface | + | ==/etc/network/interface== |
| + | <pre> | ||
| + | auto lo | ||
| + | iface lo inet loopback | ||
| − | + | iface eth2 inet manual | |
| − | + | auto eth3 | |
| − | + | iface eth3 inet static | |
| − | |||
| − | |||
| − | |||
| − | |||
| − | |||
address 10.14.14.52 | address 10.14.14.52 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| − | + | # Connected to Node2 (.51) | |
| − | + | auto eth0 | |
| + | iface eth0 inet static | ||
address 10.15.15.52 | address 10.15.15.52 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| Line 129: | Line 132: | ||
down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0 | down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0 | ||
| − | + | # Connected to Node1 (.50) | |
| − | + | auto eth1 | |
| + | iface eth1 inet static | ||
address 10.15.15.52 | address 10.15.15.52 | ||
netmask 255.255.255.0 | netmask 255.255.255.0 | ||
| Line 136: | Line 140: | ||
down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1 | down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1 | ||
| − | + | auto vmbr0 | |
| − | + | iface vmbr0 inet static | |
address 192.168.2.52 | address 192.168.2.52 | ||
netmask 255.255.240.0 | netmask 255.255.240.0 | ||
| Line 144: | Line 148: | ||
bridge_stp off | bridge_stp off | ||
bridge_fd 0 | bridge_fd 0 | ||
| + | </pre> | ||
| + | |||
==route== | ==route== | ||
| − | + | <pre> | |
root@pve-2-52:~# route -n | root@pve-2-52:~# route -n | ||
| − | + | Kernel IP routing table | |
| − | + | Destination Gateway Genmask Flags Metric Ref Use Iface | |
| − | + | 10.15.15.51 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 | |
| − | + | 10.15.15.50 0.0.0.0 255.255.255.255 UH 0 0 0 eth1 | |
| − | + | 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 | |
| − | + | 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 | |
| − | + | 10.14.14.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3 | |
| − | + | 192.168.0.0 0.0.0.0 255.255.240.0 U 0 0 0 vmbr0 | |
| − | + | 0.0.0.0 192.168.2.1 0.0.0.0 UG 0 0 0 vmbr0 | |
| + | </pre> | ||
[[Category: HOWTO]] | [[Category: HOWTO]] | ||
Revision as of 11:49, 28 October 2016
Introduction
This wiki page describe a three node Ceph Server network configuration, using full mesh network (instead of a network switch). This should also work with a 5-node cluster, general you need nodes_total - 1 = nic ports. The basic idea is running a small 3 node cluster with 10 Gbit network WITHOUT buying an expensive 10 Gbit network switch.
This should work with 40 Gbit or even 100 Gbit Nics. But for this howto, 10 Gbit Intel NICs were used.
Node1
/etc/network/interface
auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
address 10.14.14.50
netmask 255.255.255.0
# Connected to Node2 (.51)
auto eth0
iface eth0 inet static
address 10.15.15.50
netmask 255.255.255.0
up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
# Connected to Node3 (.52)
auto eth1
iface eth1 inet static
address 10.15.15.50
netmask 255.255.255.0
up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
address 192.168.2.50
netmask 255.255.240.0
gateway 192.168.2.1
bridge_ports eth2
bridge_stp off
bridge_fd 0
route
root@pve-2-50:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 10.15.15.51 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 10.15.15.52 0.0.0.0 255.255.255.255 UH 0 0 0 eth1 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 10.14.14.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3 192.168.0.0 0.0.0.0 255.255.240.0 U 0 0 0 vmbr0 0.0.0.0 192.168.2.1 0.0.0.0 UG 0 0 0 vmbr0
Node2
/etc/network/interface
auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
address 10.14.14.51
netmask 255.255.255.0
# Connected to Node1 (.50)
auto eth0
iface eth0 inet static
address 10.15.15.51
netmask 255.255.255.0
up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth0
down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0
# Connected to Node3 (.52)
auto eth1
iface eth1 inet static
address 10.15.15.51
netmask 255.255.255.0
up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
address 192.168.2.51
netmask 255.255.240.0
gateway 192.168.2.1
bridge_ports eth2
bridge_stp off
bridge_fd 0
route
root@pve-2-51:/# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 10.15.15.50 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 10.15.15.52 0.0.0.0 255.255.255.255 UH 0 0 0 eth1 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 10.14.14.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3 192.168.0.0 0.0.0.0 255.255.240.0 U 0 0 0 vmbr0 0.0.0.0 192.168.2.1 0.0.0.0 UG 0 0 0 vmbr0
Node3
/etc/network/interface
auto lo
iface lo inet loopback
iface eth2 inet manual
auto eth3
iface eth3 inet static
address 10.14.14.52
netmask 255.255.255.0
# Connected to Node2 (.51)
auto eth0
iface eth0 inet static
address 10.15.15.52
netmask 255.255.255.0
up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
# Connected to Node1 (.50)
auto eth1
iface eth1 inet static
address 10.15.15.52
netmask 255.255.255.0
up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth1
down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1
auto vmbr0
iface vmbr0 inet static
address 192.168.2.52
netmask 255.255.240.0
gateway 192.168.2.1
bridge_ports eth2
bridge_stp off
bridge_fd 0
route
root@pve-2-52:~# route -n Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 10.15.15.51 0.0.0.0 255.255.255.255 UH 0 0 0 eth0 10.15.15.50 0.0.0.0 255.255.255.255 UH 0 0 0 eth1 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 10.15.15.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1 10.14.14.0 0.0.0.0 255.255.255.0 U 0 0 0 eth3 192.168.0.0 0.0.0.0 255.255.240.0 U 0 0 0 vmbr0 0.0.0.0 192.168.2.1 0.0.0.0 UG 0 0 0 vmbr0