Hi,
Outgoing traffic (from a jail) via PF NAT over a LAGG/LACP sometimes has
the *backup* CARP IP address assigned to it.
Obivously as this IP is only active on the "other" server, the return
TCP connection traffic never actually gets back to our CARP master, and
the other server sees spurious TCP connections. This is very
reproducible and appears to be deterministic, like a round robin IP
allocation. In practice, inside a jail, `curl $URL` will fail
repeatedly.
Hopefully this is some misconfiguration on my part - what am I doing
wrong?
BTW I wrote this up a while back on the forums where the config files
are easier to read: https://forums.freebsd.org/threads/61552
###
# /etc/rc.conf network
ifconfig_igb0="up"
ifconfig_igb1="up"
cloned_interfaces="${cloned_interfaces} lagg0"
defaultrouter="1.2.3.81"
ipv6_defaultrouter="1:2:3:4::1"
ifconfig_lagg0="inet 1.2.3.83/28 laggproto lacp laggport igb0 laggport
igb1"
ifconfig_lagg0_ipv6="inet6 1:2:3:4::83/64"
# carp on
kld_list="${kld_list} carp"
ifconfig_lagg0_aliases="\
inet vhid 1 advskew 100 pass pw1 1.2.3.84/32 \
inet6 vhid 2 advskew 100 pass pw2 1:2:3:4::84/64 \
inet vhid 3 advskew 0 pass pw3 1.2.3.85/32 \
inet6 vhid 4 advskew 0 pass pw4 1:2:3:4::85/64 \
"
# jail networks use their own separate cloned if
cloned_interfaces="${cloned_interfaces} lo1"
ifconfig_lo1_aliases="inet 10.241.0.0-15/16"
###
# /etc/pf.conf
# macros
protocols = "{ tcp, udp, icmp }"
# interfaces
extl_if="lagg0"
intl_if="lo0"
jail_if="lo1"
# networks
intl_net = $intl_if:network
jail_net = $jail_if:network
internet = $extl_if:network
# limits
set limit { states 20, frags 8, src-nodes 8 }
set timeout { adaptive.start 18, adaptive.end 20 }
# clean packets are happy packets
scrub in all
# jails are allowed outbound connections but not inbound
nat on $extl_if proto $protocols from $jail_net to any -> ($extl_if)
# o ye of little faith
pass in all
pass out all
###
running configs ##
pfctl indeed shows its a round-robin
###
# pfctl -vnf /etc/pf.conf
protocols = "{ tcp, udp, icmp }"
extl_if = "lagg0"
intl_if = "lo0"
jail_if = "lo1"
intl_net = "lo0:network"
jail_net = "lo1:network"
internet = "lagg0:network"
set limit states 20
set limit frags 8
set limit src-nodes 8
set timeout adaptive.start 18
set timeout adaptive.end 20
scrub in all fragment reassemble
nat on lagg0 inet proto tcp from 10.241.0.0/16 to any -> (lagg0)
round-robin
nat on lagg0 inet proto tcp from 10.241.0.1 to any -> (lagg0)
round-robin
... repeated for each IP
###
# ifconfig
lagg0: flags=8943 metric
0 mtu 1500
options=6403bb
ether 78:45:c4:fa:d2:99
inet 1.2.3.82 netmask 0xfff0 broadcast 1.2.3.95
* inet 1.2.3.84 netmask 0x broadcast 1.2.3.84 vhid 1
* inet 1.2.3.85 netmask 0x broadcast 1.2.3.85 vhid 3
inet6 fe80::7a45:c4ff:fefa:d299%lagg0 prefixlen 64 scopeid 0x4
inet6 1:2:3:4::82 prefixlen 64
* inet6 1:2:3:4::84 prefixlen 64 vhid 2
* inet6 1:2:3:4::85 prefixlen 64 vhid 4
nd6 options=21
media: Ethernet autoselect
status: active
* carp: MASTER vhid 1 advbase 1 advskew 0
* carp: BACKUP vhid 3 advbase 1 advskew 100
* carp: MASTER vhid 2 advbase 1 advskew 0
* carp: BACKUP vhid 4 advbase 1 advskew 100
groups: lagg
laggproto lacp lagghash l2,l3,l4
* laggport: igb0 flags=1c
* laggport: igb1 flags=1c
# I removed the lines appended with !!!.. so that the system
actually works atm
___
freebsd-pf@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/freebsd-pf
To unsubscribe, send any mail to "freebsd-pf-unsubscr...@freebsd.org"
