Hello,
I've discovered a bug in the bonding module of the Linux Kernel, which
appears
only in bonding-mode balance-alb.
Description:
You have to setup a box with at least two NICs, a bonding device
enslaving
those, assign at least two IPs to the bond and make some traffic from a
different machine to one of those IPs.
If you delete that IP, the box will regardlessly send ARP-replies to the
machine which communicated to that IP before removing it.
This comes from the rx_hashtbl and the receive load balancing algorithm.
The bug is very serious if bonding is used in a cluster-environment
using
two nodes which are connected to the same subnet. If an IP-bound service
has
to failover to the other node, the old node would announce its
MAC-address
for the IP which isn't owned by the node anymore. So client-traffic in
the
same net would hit the old node.
A possible workaround could be the usage of balance-tlb instead of
balance-alb.
I've made a little patch which removes every entry from the rx_hashtbl, if
the
according IP is removed from the bond. The patch was made for Linux Kernel
version 2.6.19.
---8<---
diff -ur linux-2.6.19/drivers/net/bonding/bond_alb.c
linux/drivers/net/bonding/bond_alb.c
--- linux-2.6.19/drivers/net/bonding/bond_alb.c 2006-11-29
22:57:37.000000000 +0100
+++ linux/drivers/net/bonding/bond_alb.c 2007-01-16
17:23:53.000000000 +0100
@@ -1677,3 +1677,38 @@
}
}
+void bond_alb_remove_ip_from_rlb(struct bonding *bond, u32 ip) {
+ struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
+ u32 curr_index;
+
+ dprintk("%s: removing entries from rx_hashtbl for IP %lx\n",
bond->dev->name, ip);
+ _lock_rx_hashtbl(bond);
+
+ curr_index = bond_info->rx_hashtbl_head;
+ while (curr_index != RLB_NULL_INDEX) {
+ struct rlb_client_info *curr =
&(bond_info->rx_hashtbl[curr_index]);
+ u32 next_index = bond_info->rx_hashtbl[curr_index].next;
+ u32 prev_index = bond_info->rx_hashtbl[curr_index].prev;
+
+ if (curr->ip_src == ip) {
+ dprintk("%s: entry %u matched\n", bond->dev->name,
curr_index);
+
+ if (curr_index == bond_info->rx_hashtbl_head) {
+ bond_info->rx_hashtbl_head = next_index;
+ }
+ if (prev_index != RLB_NULL_INDEX) {
+ bond_info->rx_hashtbl[prev_index].next =
next_index;
+ }
+ if (next_index != RLB_NULL_INDEX) {
+ bond_info->rx_hashtbl[next_index].prev =
prev_index;
+ }
+
+ rlb_init_table_entry(curr);
+ }
+
+ curr_index = next_index;
+ }
+
+ _unlock_rx_hashtbl(bond);
+}
+
diff -ur linux-2.6.19/drivers/net/bonding/bond_alb.h
linux/drivers/net/bonding/bond_alb.h
--- linux-2.6.19/drivers/net/bonding/bond_alb.h 2006-11-29
22:57:37.000000000 +0100
+++ linux/drivers/net/bonding/bond_alb.h 2007-01-16
17:23:53.000000000 +0100
@@ -128,5 +128,6 @@
void bond_alb_monitor(struct bonding *bond);
int bond_alb_set_mac_address(struct net_device *bond_dev, void *addr);
void bond_alb_clear_vlan(struct bonding *bond, unsigned short vlan_id);
+void bond_alb_remove_ip_from_rlb(struct bonding *bond, u32 ip);
#endif /* __BOND_ALB_H__ */
diff -ur linux-2.6.19/drivers/net/bonding/bond_main.c
linux/drivers/net/bonding/bond_main.c
--- linux-2.6.19/drivers/net/bonding/bond_main.c 2006-11-29
22:57:37.000000000 +0100
+++ linux/drivers/net/bonding/bond_main.c 2007-01-16
17:30:49.000000000 +0100
@@ -3356,6 +3356,12 @@
return NOTIFY_OK;
case NETDEV_DOWN:
bond->master_ip =
bond_glean_dev_ip(bond->dev);
+
+ /* remove IP from RLB hashtable if using
balance-alb mode: */
+ if (bond->params.mode == BOND_MODE_ALB) {
+ bond_alb_remove_ip_from_rlb(bond,
ifa->ifa_local);
+ }
+
return NOTIFY_OK;
default:
return NOTIFY_DONE;
---8<---
The function bond_alb_remove_ip_from_rlb is heavily based on the function
rlb_clear_vlan.
And here's a useful patch for debugging purposes (it outputs the rx_hashtbl
in
the proc-file of the bond):
---8<---
diff -ur linux-2.6.19/drivers/net/bonding/bond_alb.c
linux/drivers/net/bonding/bond_alb.c
--- linux-2.6.19/drivers/net/bonding/bond_alb.c 2007-01-16
18:59:32.000000000 +0100
+++ linux/drivers/net/bonding/bond_alb.c 2007-01-16
18:48:15.000000000 +0100
@@ -26,6 +26,7 @@
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/pkt_sched.h>
+#include <linux/seq_file.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/timer.h>
@@ -1677,6 +1678,45 @@
}
}
+void bond_alb_info_show(struct seq_file *seq) {
+ struct bonding *bond = seq->private;
+ struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
+ struct rlb_client_info *rx_hash_table;
+ u32 index;
+ u32 src, dst;
+
+ seq_puts(seq, "\nALB info\n\n");
+ seq_puts(seq, " Receive Load Balancing table:\n\n");
+ seq_puts(seq, " Index Slave Server Client
Client-MAC Asgnd\n");
+
+ _lock_rx_hashtbl(bond);
+
+ rx_hash_table = bond_info->rx_hashtbl;
+
+ if (rx_hash_table != NULL) {
+ for (index = bond_info->rx_hashtbl_head;
+ index != RLB_NULL_INDEX;
+ index = rx_hash_table[index].next) {
+ src = ntohl(rx_hash_table[index].ip_src);
+ dst = ntohl(rx_hash_table[index].ip_dst);
+
+ seq_printf(seq,
+ " %03u %8s %03u.%03u.%03u.%03u
%03u.%03u.%03u.%03u %02x:%02x:%02x:%02x:%02x:%02x %3s\n",
+ index,
+ (rx_hash_table[index].slave != NULL
? rx_hash_table[index].slave->dev->name : "none"),
+ ((src >> 24) & 0xff), ((src >> 16) &
0xff), ((src >> 8) & 0xff), (src & 0xff),
+ ((dst >> 24) & 0xff), ((dst >> 16) &
0xff), ((dst >> 8) & 0xff), (dst & 0xff),
+ rx_hash_table[index].mac_dst[0],
rx_hash_table[index].mac_dst[1],
+ rx_hash_table[index].mac_dst[2],
rx_hash_table[index].mac_dst[3],
+ rx_hash_table[index].mac_dst[4],
rx_hash_table[index].mac_dst[5],
+ (rx_hash_table[index].assigned ?
"yes" : "no")
+ );
+ }
+ }
+
+ _unlock_rx_hashtbl(bond);
+}
+
void bond_alb_remove_ip_from_rlb(struct bonding *bond, u32 ip) {
struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond));
u32 curr_index;
diff -ur linux-2.6.19/drivers/net/bonding/bond_alb.h
linux/drivers/net/bonding/bond_alb.h
--- linux-2.6.19/drivers/net/bonding/bond_alb.h 2007-01-16
18:59:32.000000000 +0100
+++ linux/drivers/net/bonding/bond_alb.h 2007-01-16
19:01:46.000000000 +0100
@@ -128,6 +128,7 @@
void bond_alb_monitor(struct bonding *bond);
int bond_alb_set_mac_address(struct net_device *bond_dev, void *addr);
void bond_alb_clear_vlan(struct bonding *bond, unsigned short vlan_id);
+void bond_alb_info_show(struct seq_file *seq);
void bond_alb_remove_ip_from_rlb(struct bonding *bond, u32 ip);
#endif /* __BOND_ALB_H__ */
diff -ur linux-2.6.19/drivers/net/bonding/bond_main.c
linux/drivers/net/bonding/bond_main.c
--- linux-2.6.19/drivers/net/bonding/bond_main.c 2007-01-16
18:59:32.000000000 +0100
+++ linux/drivers/net/bonding/bond_main.c 2007-01-16
18:48:15.000000000 +0100
@@ -3048,6 +3048,10 @@
ad_info.partner_system[5]);
}
}
+ else
+ if (bond->params.mode == BOND_MODE_ALB) {
+ bond_alb_info_show(seq);
+ }
}
static void bond_info_show_slave(struct seq_file *seq, const struct slave
*slave)
---8<---
I attach this example to visualize the bug. The box is named 'linux' (which
has
the two IPs 10.0.91.128 and 10.0.91.129) and the other machine (which makes
some traffic) is called 'dave'. Their clocks are synchronized via NTP.
---8<---
linux:~ # modprobe bonding miimon=100 updelay=200 mode=balance-alb
use_carrier=0
linux:~ # ifconfig bond0 10.0.91.128 netmask 255.255.255.0 up
linux:~ # ifenslave bond0 eth1
linux:~ # ifenslave bond0 eth2
linux:~ # ip addr add 10.0.91.129 dev bond0
linux:~ # ip addr sh bond0
18: bond0: <BROADCAST,MULTICAST,MASTER,UP> mtu 1500 qdisc noqueue
link/ether 00:02:b3:55:2e:b1 brd ff:ff:ff:ff:ff:ff
inet 10.0.91.128/24 brd 10.255.255.255 scope global bond0
inet 10.0.91.129/32 scope global bond0
inet6 fe80::200:ff:fe00:0/64 scope link
valid_lft forever preferred_lft forever
---
dave:~ # ping 10.0.91.129
PING 10.0.91.129 (10.0.91.129) 56(84) bytes of data.
64 bytes from 10.0.91.129: icmp_seq=1 ttl=64 time=3.83 ms
64 bytes from 10.0.91.129: icmp_seq=2 ttl=64 time=0.205 ms
[...]
dave:~ # tcpdump -i bond0 arp host 10.0.91.129
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on bond0, link-type EN10MB (Ethernet), capture size 96 bytes
11:55:41.829735 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:55:41.830993 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:55:44.047261 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:55:44.047276 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
[...]
---
linux:~ # ip addr del 10.0.91.129 dev bond0
linux:~ # ip addr sh bond0
18: bond0: <BROADCAST,MULTICAST,MASTER,UP> mtu 1500 qdisc noqueue
link/ether 00:02:b3:55:2e:b1 brd ff:ff:ff:ff:ff:ff
inet 10.0.91.128/24 brd 10.255.255.255 scope global bond0
inet6 fe80::200:ff:fe00:0/64 scope link
valid_lft forever preferred_lft forever
linux:~ # date
Tue Jan 16 11:55:57 CET 2007
---
dave:~ # date
Tue Jan 16 11:56:59 CET 2007
dave:~ # tcpdump -i bond0 arp host 10.0.91.129
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on bond0, link-type EN10MB (Ethernet), capture size 96 bytes
11:57:04.305078 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:57:04.306248 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:57:06.704552 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
11:57:06.704569 arp reply 10.0.91.129 is-at 00:02:b3:55:2e:b1 (oui Unknown)
[...]
---8<---
Bye
Christian Jung
PS I'm sorry but I have to use a mailer which has some handicaps. If the
whitespaces of the patches are munged in any way I can send you the patches
as
attachment.
Another thing: When shutting down a bond (e.g. ifconfig bond0 0.0.0.0 down)
the
slaves keep the master IP address of the bond. Is there a special reason for
this behaviour?
phone: +49 6898/10-4987
fax: +49 6898/10-54987
http://www.saarstahl.de
-
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
