Difference between revisions of "Full Mesh Network for Ceph Server"

From Proxmox VE
Jump to navigation Jump to search
m (wikiformat + comments)
Line 7: Line 7:
 
=Node1=
 
=Node1=
 
==/etc/network/interface==
 
==/etc/network/interface==
 
 
<pre>
 
<pre>
 
auto lo
 
auto lo
Line 19: Line 18:
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
 +
# Connected to Node2 (.51)
 
auto eth0
 
auto eth0
 
iface eth0 inet static
 
iface eth0 inet static
Line 26: Line 26:
 
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
 
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
  
 +
# Connected to Node3 (.52)
 
auto eth1
 
auto eth1
 
iface eth1 inet static
 
iface eth1 inet static
Line 58: Line 59:
  
 
=Node2=
 
=Node2=
==/etc/network/interface/==
+
==/etc/network/interface==
 +
<pre>
 +
auto lo
 +
iface lo inet loopback
  
root@pve-2-51:/# cat /etc/network/interfaces
+
iface eth2 inet manual
  
auto lo
+
auto eth3
iface lo inet loopback
+
iface eth3 inet static
 
 
iface eth2 inet manual
 
 
 
auto eth3
 
iface eth3 inet static
 
 
         address  10.14.14.51
 
         address  10.14.14.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
auto eth0
+
# Connected to Node1 (.50)
iface eth0 inet static
+
auto eth0
 +
iface eth0 inet static
 
         address  10.15.15.51
 
         address  10.15.15.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
Line 79: Line 79:
 
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0
 
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0
  
auto eth1
+
# Connected to Node3 (.52)
iface eth1 inet static
+
auto eth1
 +
iface eth1 inet static
 
         address  10.15.15.51
 
         address  10.15.15.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
Line 86: Line 87:
 
         down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
 
         down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
  
auto vmbr0
+
auto vmbr0
iface vmbr0 inet static
+
iface vmbr0 inet static
 
         address  192.168.2.51
 
         address  192.168.2.51
 
         netmask  255.255.240.0
 
         netmask  255.255.240.0
Line 94: Line 95:
 
         bridge_stp off
 
         bridge_stp off
 
         bridge_fd 0
 
         bridge_fd 0
 +
</pre>
  
 
==route==
 
==route==
 +
<pre>
 
root@pve-2-51:/# route -n
 
root@pve-2-51:/# route -n
Kernel IP routing table
+
Kernel IP routing table
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
+
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
10.15.15.50    0.0.0.0        255.255.255.255 UH    0      0        0 eth0
+
10.15.15.50    0.0.0.0        255.255.255.255 UH    0      0        0 eth0
10.15.15.52    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
+
10.15.15.52    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth0
+
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth0
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
+
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
+
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
192.168.0.0    0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
+
192.168.0.0    0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
+
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
 +
</pre>
  
 
=Node3=
 
=Node3=
==/etc/network/interface/==
+
==/etc/network/interface==
 +
<pre>
 +
auto lo
 +
iface lo inet loopback
  
root@pve-2-52:~# cat /etc/network/interfaces
+
iface eth2 inet manual
  
auto lo
+
auto eth3
iface lo inet loopback
+
iface eth3 inet static
 
 
iface eth2 inet manual
 
 
 
auto eth3
 
iface eth3 inet static
 
 
         address  10.14.14.52
 
         address  10.14.14.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
auto eth0
+
# Connected to Node2 (.51)
iface eth0 inet static
+
auto eth0
 +
iface eth0 inet static
 
         address  10.15.15.52
 
         address  10.15.15.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
Line 129: Line 132:
 
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
 
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
  
auto eth1
+
# Connected to Node1 (.50)
iface eth1 inet static
+
auto eth1
 +
iface eth1 inet static
 
         address  10.15.15.52
 
         address  10.15.15.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
Line 136: Line 140:
 
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1
 
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1
  
auto vmbr0
+
auto vmbr0
iface vmbr0 inet static
+
iface vmbr0 inet static
 
         address  192.168.2.52
 
         address  192.168.2.52
 
         netmask  255.255.240.0
 
         netmask  255.255.240.0
Line 144: Line 148:
 
         bridge_stp off
 
         bridge_stp off
 
         bridge_fd 0
 
         bridge_fd 0
 +
</pre>
 +
 
==route==
 
==route==
 
+
<pre>
 
root@pve-2-52:~# route -n
 
root@pve-2-52:~# route -n
Kernel IP routing table
+
Kernel IP routing table
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
+
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
10.15.15.51    0.0.0.0        255.255.255.255 UH    0      0        0 eth0
+
10.15.15.51    0.0.0.0        255.255.255.255 UH    0      0        0 eth0
10.15.15.50    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
+
10.15.15.50    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth0
+
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth0
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
+
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
+
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
192.168.0.0    0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
+
192.168.0.0    0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
+
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
 +
</pre>
  
 
[[Category: HOWTO]]
 
[[Category: HOWTO]]

Revision as of 13:49, 28 October 2016

Introduction

This wiki page describe a three node Ceph Server network configuration, using full mesh network (instead of a network switch). This should also work with a 5-node cluster, general you need nodes_total - 1 = nic ports. The basic idea is running a small 3 node cluster with 10 Gbit network WITHOUT buying an expensive 10 Gbit network switch.

This should work with 40 Gbit or even 100 Gbit Nics. But for this howto, 10 Gbit Intel NICs were used.

Node1

/etc/network/interface

auto lo
iface lo inet loopback

iface eth2 inet manual

auto eth3
iface eth3 inet static
        address  10.14.14.50
        netmask  255.255.255.0

# Connected to Node2 (.51)
auto eth0
iface eth0 inet static
        address  10.15.15.50
        netmask  255.255.255.0
        up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
        down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0

# Connected to Node3 (.52)
auto eth1
iface eth1 inet static
        address  10.15.15.50
        netmask  255.255.255.0
        up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
        down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.50
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports eth2
        bridge_stp off
        bridge_fd 0

route

root@pve-2-50:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.51     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.52     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0

Node2

/etc/network/interface

auto lo
iface lo inet loopback

iface eth2 inet manual

auto eth3
iface eth3 inet static
        address  10.14.14.51
        netmask  255.255.255.0

# Connected to Node1 (.50)
auto eth0
iface eth0 inet static
        address  10.15.15.51
        netmask  255.255.255.0
        up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth0
        down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0

# Connected to Node3 (.52)
auto eth1
iface eth1 inet static
        address  10.15.15.51
        netmask  255.255.255.0
        up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
        down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.51
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports eth2
        bridge_stp off
        bridge_fd 0

route

root@pve-2-51:/# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.50     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.52     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0

Node3

/etc/network/interface

auto lo
iface lo inet loopback

iface eth2 inet manual

auto eth3
iface eth3 inet static
        address  10.14.14.52
        netmask  255.255.255.0

# Connected to Node2 (.51)
auto eth0
iface eth0 inet static
        address  10.15.15.52
        netmask  255.255.255.0
        up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
        down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0

# Connected to Node1 (.50)
auto eth1
iface eth1 inet static
        address  10.15.15.52
        netmask  255.255.255.0
        up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth1
        down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.52
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports eth2
        bridge_stp off
        bridge_fd 0

route

root@pve-2-52:~# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.15.15.51     0.0.0.0         255.255.255.255 UH    0      0        0 eth0
10.15.15.50     0.0.0.0         255.255.255.255 UH    0      0        0 eth1
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth0
10.15.15.0      0.0.0.0         255.255.255.0   U     0      0        0 eth1
10.14.14.0      0.0.0.0         255.255.255.0   U     0      0        0 eth3
192.168.0.0     0.0.0.0         255.255.240.0   U     0      0        0 vmbr0
0.0.0.0         192.168.2.1     0.0.0.0         UG    0      0        0 vmbr0