Now that vrf leaking is supported with default vrf, setup is more simple for exit node. (not need extra interface)
Also cleanup symmetric config Signed-off-by: Alexandre Derumier <aderum...@odiso.com> --- vxlan-and-evpn.adoc | 239 +++++++++++--------------------------------- 1 file changed, 59 insertions(+), 180 deletions(-) diff --git a/vxlan-and-evpn.adoc b/vxlan-and-evpn.adoc index 9cd55fc..bb2a66d 100644 --- a/vxlan-and-evpn.adoc +++ b/vxlan-and-evpn.adoc @@ -366,7 +366,6 @@ iface vmbr3 inet manual ---- router bgp 1234 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.2 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -435,7 +434,6 @@ iface vmbr3 inet manual ---- router bgp 1234 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -505,7 +503,6 @@ iface vmbr3 inet manual ---- router bgp 1234 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.2 remote-as 1234 @@ -612,7 +609,6 @@ frr.conf ---- router bgp 1234 bgp router-id 192.168.0.1 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.2 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -689,7 +685,6 @@ frr.conf ---- router bgp 1234 bgp router-id 192.168.0.2 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -766,7 +761,6 @@ frr.conf ---- router bgp 1234 bgp router-id 192.168.0.3 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.2 remote-as 1234 @@ -879,7 +873,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:90 #must be different on each node vrf vrf1 ---- @@ -888,10 +881,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf ! router bgp 1234 bgp router-id 192.168.0.1 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.2 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -902,18 +895,6 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 192.168.0.1 - ! - address-family ipv4 unicast - redistribute connected - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! line vty ! ---- @@ -992,7 +973,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:91 #must be different on each node vrf vrf1 ---- @@ -1002,10 +982,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf ! router bgp 1234 bgp router-id 192.168.0.2 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -1016,18 +996,6 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 192.168.0.2 - ! - address-family ipv4 unicast - redistribute connected - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! line vty ! ---- @@ -1106,7 +1074,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:92 #must be different on each node vrf vrf1 ---- @@ -1116,10 +1083,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf ! router bgp 1234 bgp router-id 192.168.0.3 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.2 remote-as 1234 @@ -1130,18 +1097,6 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 192.168.0.3 - ! - address-family ipv4 unicast - redistribute connected - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! line vty ! ---- @@ -1153,8 +1108,7 @@ Routing to outside need the symmetric model. 1 gateway node ^^^^^^^^^^^^^^ In this example, we'll use only 1 proxmox node as exit gateway. (node1) -This node have a simple default gw in the vrf to the external router (no bgp between router and node1) -and announce this default gw to other proxmox nodes. +This node announce the default gw in vrf1 (default originate) and forward to his own default gateway (192.168.0.254) (no bgp between router and node1) *node1 @@ -1172,19 +1126,11 @@ auto vmbr0 iface vmbr0 inet static address 192.168.0.1 netmask 255.255.255.0 + gateway 192.168.0.254 bridge_ports eno1 bridge_stp off bridge_fd 0 -auto eno2 -iface eno2 - address 172.16.0.1 - netmask 255.255.255.0 - vrf vrf1 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - #if you have multiple external routers, you can use ecmp balancing - #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1 - auto vxlan2 iface vxlan2 inet manual vxlan-id 2 @@ -1238,7 +1184,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:90 #must be different on each node vrf vrf1 ---- @@ -1248,10 +1193,13 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf +! +ip route 10.0.2.0/24 vmbr2 nexthop-vrf vrf1 +ip route 10.0.3.0/24 vmbr3 nexthop-vrf vrf1 ! router bgp 1234 bgp router-id 192.168.0.1 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.2 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -1264,15 +1212,8 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! - bgp router-id 172.16.0.1 - ! - address-family ipv4 unicast - redistribute connected - redistribute kernel !announce your default gw to all nodes - exit-address-family - ! address-family l2vpn evpn - advertise ipv4 unicast + default-originate ipv4 exit-address-family ! line vty @@ -1353,7 +1294,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:91 #must be different on each node vrf vrf1 ---- @@ -1363,10 +1303,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf ! router bgp 1234 bgp router-id 192.168.0.2 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.3 remote-as 1234 @@ -1377,18 +1317,6 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 192.168.0.2 - ! - address-family ipv4 unicast - redistribute connected - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! line vty ! ---- @@ -1467,7 +1395,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:92 #must be different on each node vrf vrf1 ---- @@ -1477,10 +1404,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf ! router bgp 1234 bgp router-id 192.168.0.3 - no bgp default ipv4-unicast coalesce-time 1000 neighbor 192.168.0.1 remote-as 1234 neighbor 192.168.0.2 remote-as 1234 @@ -1491,18 +1418,6 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 192.168.0.3 - ! - address-family ipv4 unicast - redistribute connected - exit-address-family - ! - address-family l2vpn evpn - advertise ipv4 unicast - exit-address-family -! line vty ! ---- @@ -1510,8 +1425,8 @@ line vty multiple gateway nodes ^^^^^^^^^^^^^^^^^^^^^^ In this example, all nodes will be used as exit gateway. (But you can use only 2 nodes if you want) -All nodes have a simple default gw in the vrf to the external router (no bgp between router and node1) -and announce this default gw. +All nodes have a a default gw to the external router (192.168.0.254) (no bgp between router and node1) +and announce this default gw in the vrf (default originate) The external router have ecmp routes to all proxmox nodes.(balancing). If the router send the packet to a wrong node (vm is not on this node), this node will route through vxlan the packet to final destination. @@ -1531,20 +1446,11 @@ auto vmbr0 iface vmbr0 inet static address 192.168.0.1 netmask 255.255.255.0 + gateway 192.168.0.254 bridge_ports eno1 bridge_stp off bridge_fd 0 -auto eno2 -iface eno2 - address 172.16.0.1 - netmask 255.255.255.0 - vrf vrf1 - mtu 1550 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - #if you have multiple external routers, you can use ecmp balancing - #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1 - auto vxlan2 iface vxlan2 inet manual vxlan-id 2 @@ -1598,7 +1504,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:90 #must be different on each node vrf vrf1 ---- @@ -1608,6 +1513,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf +! +ip route 10.0.2.0/24 vmbr2 nexthop-vrf vrf1 +ip route 10.0.3.0/24 vmbr3 nexthop-vrf vrf1 ! router bgp 1234 bgp router-id 192.168.0.1 @@ -1624,15 +1533,8 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! - bgp router-id 172.16.0.1 - ! - address-family ipv4 unicast - redistribute connected - redistribute kernel !announce your default gw to all nodes - exit-address-family - ! address-family l2vpn evpn - advertise ipv4 unicast + default-originate ipv4 exit-address-family ! line vty @@ -1655,20 +1557,11 @@ auto vmbr0 iface vmbr0 inet static address 192.168.0.2 netmask 255.255.255.0 + gateway 192.168.0.254 bridge_ports eno1 bridge_stp off bridge_fd 0 -auto eno2 -iface eno2 - address 172.16.0.3 - netmask 255.255.255.0 - vrf vrf1 - mtu 1550 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - #if you have multiple external routers, you can use ecmp balancing - #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1 - auto vxlan2 iface vxlan2 inet manual vxlan-id 2 @@ -1723,7 +1616,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:91 #must be different on each node vrf vrf1 ---- @@ -1733,6 +1625,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf +! +ip route 10.0.2.0/24 vmbr2 nexthop-vrf vrf1 +ip route 10.0.3.0/24 vmbr3 nexthop-vrf vrf1 ! router bgp 1234 bgp router-id 192.168.0.2 @@ -1747,17 +1643,8 @@ router bgp 1234 advertise-all-vni exit-address-family ! -router bgp 1234 vrf vrf1 -! - bgp router-id 172.16.0.2 - ! - address-family ipv4 unicast - redistribute connected - redistribute kernel !announce your default gw to all nodes - exit-address-family - ! address-family l2vpn evpn - advertise ipv4 unicast + default-originate ipv4 exit-address-family ! line vty @@ -1780,20 +1667,11 @@ auto vmbr0 iface vmbr0 inet static address 192.168.0.3 netmask 255.255.255.0 + gateway 192.168.0.254 bridge_ports eno1 bridge_stp off bridge_fd 0 -auto eno2 -iface eno2 - address 172.16.0.3 - netmask 255.255.255.0 - vrf vrf1 - mtu 1550 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - #if you have multiple external routers, you can use ecmp balancing - #post-up route add default nexthop via 172.16.0.253 dev eno2 vrf vrf1 nexthop via 172.16.0.254 dev eno2 vrf vrf1 - auto vxlan2 iface vxlan2 inet manual vxlan-id 2 @@ -1848,7 +1726,6 @@ iface vmbr4000 inet manual bridge_ports vxlan4000 bridge_stp off bridge_fd 0 - hwaddress 44:39:39:FF:40:92 #must be different on each node vrf vrf1 ---- @@ -1858,6 +1735,10 @@ frr.conf ---- vrf vrf1 vni 4000 + exit-vrf +! +ip route 10.0.2.0/24 vmbr2 nexthop-vrf vrf1 +ip route 10.0.3.0/24 vmbr3 nexthop-vrf vrf1 ! router bgp 1234 bgp router-id 192.168.0.3 @@ -1874,15 +1755,8 @@ router bgp 1234 ! router bgp 1234 vrf vrf1 ! - bgp router-id 172.16.0.3 - ! - address-family ipv4 unicast - redistribute connected - redistribute kernel !announce your default gw to all nodes - exit-address-family - ! address-family l2vpn evpn - advertise ipv4 unicast + default-originate ipv4 exit-address-family ! line vty @@ -1892,41 +1766,46 @@ line vty Note ^^^^ -If your external router don't support ecmp to reach multiple proxmox nodes, +If your external router don't support ecmp static route to reach multiple proxmox nodes, you can setup an HA floating vip on proxmox nodes with vrrp -I this example, we will setup an floating 172.16.0.10 ip on node1 and node2. +In this example, we will setup an floating 192.168.0.10 ip on node1 and node2. Node1 is the primary and failover to node2 in case of failure. +This setup need vrrpd package (apt install vrrpd). +#TODO : It should be possible to do it with frr directly with last version. * node1 ---- -auto eno2 -iface eno2 - address 172.16.0.1 - netmask 255.255.255.0 - vrf vrf1 - mtu 1550 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - vrrp-id 1 - vrrp-priority 1 - vrrp-virtual-ip 172.16.0.10 +auto vmbr0 +iface vmbr0 inet static + address 192.168.0.1 + netmask 255.255.255.0 + gateway 192.168.0.254 + bridge_ports eno1 + bridge_stp off + bridge_fd 0 + vrrp-id 1 + vrrp-priority 1 + vrrp-virtual-ip 192.168.0.10 ---- * node2 ---- -auto eno2 -iface eno2 - address 172.16.0.2 - netmask 255.255.255.0 - mtu 1550 - vrf vrf1 - post-up ip route add default via 172.16.0.254 dev eno2 vrf vrf1 - vrrp-id 1 - vrrp-priority 2 - vrrp-virtual-ip 172.16.0.10 +auto vmbr0 +iface vmbr0 inet static + address 192.168.0.2 + netmask 255.255.255.0 + gateway 192.168.0.254 + bridge_ports eno1 + bridge_stp off + bridge_fd 0 + vrrp-id 1 + vrrp-priority 2 + vrrp-virtual-ip 192.168.0.10 ---- +#TODO : Documentation with bgp upstream router. -- 2.20.1 _______________________________________________ pve-devel mailing list pve-devel@pve.proxmox.com https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel