Difference between revisions of "Full Mesh Network for Ceph Server"

From Proxmox VE
Jump to: navigation, search
(added example explanation to "Method 2" and adapted to "debian stretch style", i.e. "ip route" instead of "route" and "ens..." instead of "eth...")
Line 39: Line 39:
  
 
== Method 2 ==
 
== Method 2 ==
 +
 +
=== Assumptions in the Following Example ===
 +
 +
3 servers:
 +
 +
* Node1 with IP addresses x.x.x.50
 +
 +
* Node2 with IP addresses x.x.x.51
 +
 +
* Node3 with IP addresses x.x.x.52
 +
 +
4 Network cards in each server:
 +
 +
* ens18, ens19 direct conections to the 2 other servers 10.15.15.y
 +
 +
* ens20 connection to internet (router), port at vmbr0 192.168.2.y
 +
 +
* ens21 LAN (for fileserver etc.) 10.14.14.y
 +
 +
Direct connections between servers:
 +
 +
* Node1/ens18 - Node3/ens19
 +
 +
* Node2/ens18 - Node1/ens19
 +
 +
* Node3/ens18 - Node2/ens19
 +
 +
 
=== Node1 ===
 
=== Node1 ===
 
==== /etc/network/interface ====
 
==== /etc/network/interface ====
Line 45: Line 73:
 
iface lo inet loopback
 
iface lo inet loopback
  
iface eth2 inet manual
+
iface ens20 inet manual
  
auto eth3
+
auto ens21
iface eth3 inet static
+
iface ens21 inet static
 
         address  10.14.14.50
 
         address  10.14.14.50
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
# Connected to Node2 (.51)
+
# Connected to Node3 (.52)
auto eth0
+
auto ens18
iface eth0 inet static
+
iface ens18 inet static
 
         address  10.15.15.50
 
         address  10.15.15.50
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
+
         up ip route add 10.15.15.52/32 dev ens18
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
+
         down ip route del 10.15.15.52/32
  
# Connected to Node3 (.52)
+
# Connected to Node2 (.51)
auto eth1
+
auto ens19
iface eth1 inet static
+
iface ens19 inet static
 
         address  10.15.15.50
 
         address  10.15.15.50
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
+
         up ip route add 10.15.15.51/32 dev ens19
         down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
+
         down ip route del 10.15.15.51/32
  
 
auto vmbr0
 
auto vmbr0
Line 73: Line 101:
 
         netmask  255.255.240.0
 
         netmask  255.255.240.0
 
         gateway  192.168.2.1
 
         gateway  192.168.2.1
         bridge_ports eth2
+
         bridge_ports ens20
 
         bridge_stp off
 
         bridge_stp off
 
         bridge_fd 0
 
         bridge_fd 0
 +
 
</pre>
 
</pre>
  
 
==== route ====
 
==== route ====
 
<pre>
 
<pre>
root@pve-2-50:~# route -n
+
root@pve-2-50:~# ip route
Kernel IP routing table
+
default via 192.168.2.1 dev vmbr0 onlink
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
+
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.50
10.15.15.51    0.0.0.0         255.255.255.255 UH    0      0        0 eth0
+
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.50
10.15.15.52    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
+
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.50
10.15.15.0     0.0.0.0        255.255.255.0  U    0      0        0 eth0
+
10.15.15.51 dev ens19 scope link
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
+
10.15.15.52 dev ens18 scope link
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
+
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.50
192.168.0.0     0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
 
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
 
 
</pre>
 
</pre>
  
Line 98: Line 125:
 
iface lo inet loopback
 
iface lo inet loopback
  
iface eth2 inet manual
+
iface ens20 inet manual
  
auto eth3
+
auto ens21
iface eth3 inet static
+
iface ens21 inet static
 
         address  10.14.14.51
 
         address  10.14.14.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
 
# Connected to Node1 (.50)
 
# Connected to Node1 (.50)
auto eth0
+
auto ens18
iface eth0 inet static
+
iface ens18 inet static
 
         address  10.15.15.51
 
         address  10.15.15.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth0
+
         up ip route add 10.15.15.50/32 dev ens18
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth0
+
         down ip route del 10.15.15.50/32
  
 
# Connected to Node3 (.52)
 
# Connected to Node3 (.52)
auto eth1
+
auto ens19
iface eth1 inet static
+
iface ens19 inet static
 
         address  10.15.15.51
 
         address  10.15.15.51
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.52 netmask 255.255.255.255 dev eth1
+
         up ip route add 10.15.15.52/32 dev ens19
         down route del -net 10.15.15.52 netmask 255.255.255.255 dev eth1
+
         down ip route del 10.15.15.52/32
  
 
auto vmbr0
 
auto vmbr0
Line 126: Line 153:
 
         netmask  255.255.240.0
 
         netmask  255.255.240.0
 
         gateway  192.168.2.1
 
         gateway  192.168.2.1
         bridge_ports eth2
+
         bridge_ports ens20
 
         bridge_stp off
 
         bridge_stp off
 
         bridge_fd 0
 
         bridge_fd 0
 +
 
</pre>
 
</pre>
  
 
==== route ====
 
==== route ====
 
<pre>
 
<pre>
root@pve-2-51:/# route -n
+
root@pve-2-51:/# ip route
Kernel IP routing table
+
default via 192.168.2.1 dev vmbr0 onlink
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
+
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.51
10.15.15.50    0.0.0.0         255.255.255.255 UH    0      0        0 eth0
+
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.51
10.15.15.52    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
+
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.51
10.15.15.0     0.0.0.0        255.255.255.0  U    0      0        0 eth0
+
10.15.15.50 dev ens18 scope link
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
+
10.15.15.52 dev ens19 scope link
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
+
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.51
192.168.0.0     0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
 
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
 
 
</pre>
 
</pre>
  
Line 151: Line 177:
 
iface lo inet loopback
 
iface lo inet loopback
  
iface eth2 inet manual
+
iface ens20 inet manual
  
auto eth3
+
auto ens21
iface eth3 inet static
+
iface ens21 inet static
 
         address  10.14.14.52
 
         address  10.14.14.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
  
 
# Connected to Node2 (.51)
 
# Connected to Node2 (.51)
auto eth0
+
auto ens18
iface eth0 inet static
+
iface ens18 inet static
 
         address  10.15.15.52
 
         address  10.15.15.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.51 netmask 255.255.255.255 dev eth0
+
         up ip route add 10.15.15.51/32 dev ens18
         down route del -net 10.15.15.51 netmask 255.255.255.255 dev eth0
+
         down ip route del 10.15.15.51/32
  
 
# Connected to Node1 (.50)
 
# Connected to Node1 (.50)
auto eth1
+
auto ens19
iface eth1 inet static
+
iface ens19 inet static
 
         address  10.15.15.52
 
         address  10.15.15.52
 
         netmask  255.255.255.0
 
         netmask  255.255.255.0
         up route add -net 10.15.15.50 netmask 255.255.255.255 dev eth1
+
         up ip route add 10.15.15.50/32 dev ens19
         down route del -net 10.15.15.50 netmask 255.255.255.255 dev eth1
+
         down ip route del 10.15.15.50/32
  
 
auto vmbr0
 
auto vmbr0
Line 179: Line 205:
 
         netmask  255.255.240.0
 
         netmask  255.255.240.0
 
         gateway  192.168.2.1
 
         gateway  192.168.2.1
         bridge_ports eth2
+
         bridge_ports ens20
 
         bridge_stp off
 
         bridge_stp off
 
         bridge_fd 0
 
         bridge_fd 0
Line 186: Line 212:
 
==== route ====
 
==== route ====
 
<pre>
 
<pre>
root@pve-2-52:~# route -n
+
root@pve-2-52:~# ip route
Kernel IP routing table
+
default via 192.168.2.1 dev vmbr0 onlink
Destination    Gateway        Genmask        Flags Metric Ref    Use Iface
+
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.52
10.15.15.51    0.0.0.0         255.255.255.255 UH    0      0        0 eth0
+
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.52
10.15.15.50    0.0.0.0        255.255.255.255 UH    0      0        0 eth1
+
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.52
10.15.15.0     0.0.0.0        255.255.255.0  U    0      0        0 eth0
+
10.15.15.50 dev ens19 scope link
10.15.15.0      0.0.0.0        255.255.255.0  U    0      0        0 eth1
+
10.15.15.51 dev ens18 scope link
10.14.14.0      0.0.0.0        255.255.255.0  U    0      0        0 eth3
+
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.52
192.168.0.0     0.0.0.0        255.255.240.0  U    0      0        0 vmbr0
 
0.0.0.0        192.168.2.1    0.0.0.0        UG    0      0        0 vmbr0
 
 
</pre>
 
</pre>
  
 
[[Category: HOWTO]] [[Category: Cluster]]
 
[[Category: HOWTO]] [[Category: Cluster]]

Revision as of 10:15, 20 June 2017

Introduction

This wiki page describes how to configure in Proxmox VE (or any other Debian based LINUX distribution) a three node "Meshed Network" (instead of a network switch) as it can be used e.g. for connecting Ceph Servers or nodes in a Proxmox VE Cluster. This should also work with a 5-node cluster, general you need nodes_total - 1 = nic ports. The basic idea is running a small 3 node cluster with 10 Gbit network WITHOUT buying an expensive 10 Gbit network switch.

This should work with any kind of ethernet NICs, i.e. also 40 Gbit or even 100 Gbit ones. But for verifying this article, 10 Gbit Intel NICs were used.

There a two possible method to achieve a full mesh. General the fist one is the recommended one, because it is easier to set up and supports multicast.

Method 1

Create a "broadcast" bond with the given interfaces on every node. This can be done over the GUI or on the command-line.

GUI

On the GUI go to the node level -> System -> Network. Then click on "Create" and select "Linux Bond". In the Wizard make your configuration without a gateway and set mode to "broadcast".

Reboot the node to activate the new network settings.

Command-Line

Add the following lines to '/etc/network/interfaces'.

auto bond<No>
iface bond<No> inet static
	address  <IP>
	netmask  <Netmask>
	slaves <Nic1> <Nic2>
	bond_miimon 100
	bond_mode broadcast
#Full Mesh

Then start the bond

ifup bond<No>

Method 2

Assumptions in the Following Example

3 servers:

  • Node1 with IP addresses x.x.x.50
  • Node2 with IP addresses x.x.x.51
  • Node3 with IP addresses x.x.x.52

4 Network cards in each server:

  • ens18, ens19 direct conections to the 2 other servers 10.15.15.y
  • ens20 connection to internet (router), port at vmbr0 192.168.2.y
  • ens21 LAN (for fileserver etc.) 10.14.14.y

Direct connections between servers:

  • Node1/ens18 - Node3/ens19
  • Node2/ens18 - Node1/ens19
  • Node3/ens18 - Node2/ens19


Node1

/etc/network/interface

auto lo
iface lo inet loopback

iface ens20 inet manual

auto ens21
iface ens21 inet static
        address  10.14.14.50
        netmask  255.255.255.0

# Connected to Node3 (.52)
auto ens18
iface ens18 inet static
        address  10.15.15.50
        netmask  255.255.255.0
        up ip route add 10.15.15.52/32 dev ens18
        down ip route del 10.15.15.52/32

# Connected to Node2 (.51)
auto ens19
iface ens19 inet static
        address  10.15.15.50
        netmask  255.255.255.0
        up ip route add 10.15.15.51/32 dev ens19
        down ip route del 10.15.15.51/32

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.50
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports ens20
        bridge_stp off
        bridge_fd 0

route

root@pve-2-50:~# ip route
default via 192.168.2.1 dev vmbr0 onlink 
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.50 
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.50 
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.50 
10.15.15.51 dev ens19 scope link 
10.15.15.52 dev ens18 scope link 
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.50 

Node2

/etc/network/interface

auto lo
iface lo inet loopback

iface ens20 inet manual

auto ens21
iface ens21 inet static
        address  10.14.14.51
        netmask  255.255.255.0

# Connected to Node1 (.50)
auto ens18
iface ens18 inet static
        address  10.15.15.51
        netmask  255.255.255.0
        up ip route add 10.15.15.50/32 dev ens18
        down ip route del 10.15.15.50/32

# Connected to Node3 (.52)
auto ens19
iface ens19 inet static
        address  10.15.15.51
        netmask  255.255.255.0
        up ip route add 10.15.15.52/32 dev ens19
        down ip route del 10.15.15.52/32

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.51
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports ens20
        bridge_stp off
        bridge_fd 0

route

root@pve-2-51:/# ip route
default via 192.168.2.1 dev vmbr0 onlink 
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.51 
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.51 
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.51 
10.15.15.50 dev ens18 scope link 
10.15.15.52 dev ens19 scope link 
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.51 

Node3

/etc/network/interface

auto lo
iface lo inet loopback

iface ens20 inet manual

auto ens21
iface ens21 inet static
        address  10.14.14.52
        netmask  255.255.255.0

# Connected to Node2 (.51)
auto ens18
iface ens18 inet static
        address  10.15.15.52
        netmask  255.255.255.0
        up ip route add 10.15.15.51/32 dev ens18
        down ip route del 10.15.15.51/32

# Connected to Node1 (.50)
auto ens19
iface ens19 inet static
        address  10.15.15.52
        netmask  255.255.255.0
        up ip route add 10.15.15.50/32 dev ens19
        down ip route del 10.15.15.50/32

auto vmbr0
iface vmbr0 inet static
        address  192.168.2.52
        netmask  255.255.240.0
        gateway  192.168.2.1
        bridge_ports ens20
        bridge_stp off
        bridge_fd 0

route

root@pve-2-52:~# ip route
default via 192.168.2.1 dev vmbr0 onlink 
10.14.14.0/24 dev ens21 proto kernel scope link src 10.14.14.52 
10.15.15.0/24 dev ens18 proto kernel scope link src 10.15.15.52 
10.15.15.0/24 dev ens19 proto kernel scope link src 10.15.15.52 
10.15.15.50 dev ens19 scope link 
10.15.15.51 dev ens18 scope link 
192.168.0.0/20 dev vmbr0 proto kernel scope link src 192.168.2.52