Hi all,
I meet a crash problem in kernel with openvswitch2.3.0
<4>[1184369.672771] ------------[ cut here ]------------
<2>[1184369.673039] kernel BUG at include/linux/skbuff.h:1219!
<4>[1184369.673304] invalid opcode: 0000 [#1] SMP
<4>[1184369.673573] last sysfs file: /sys/devices/system/cpu/online
<4>[1184369.673838] CPU 2
<4>[1184369.673843] Modules linked in: cls_u32 sch_htb 8021q garp stp llc ipv6
openvswitch libcrc32c iptable_filter ip_tables power_meter hpilo microcode
serio_raw sg shpchp xhci_hcd ixgbe igb dca i2c_algo_bit i2c_core ext4 mbcache
jbd2 sd_mod crc_t10dif ahci dm_mirror dm_region_hash dm_log dm_mod [last
unloaded: scsi_wait_scan]
<4>[1184369.675516]
<4>[1184369.675777] Pid: 0, comm: swapper Tainted: G ---------------
H 2.6.32-279.19.31.el6.ucloud.x86_64 #1 HP ProLiant DL320e Gen8 v2
<4>[1184369.676312] RIP: 0010:[<ffffffffa01dc89d>] [<ffffffffa01dc89d>]
ovs_flow_extract+0x8ed/0xa40 [openvswitch]
<4>[1184369.676843] RSP: 0018:ffff8800456839e0 EFLAGS: 00010283
<4>[1184369.677108] RAX: ffff8807e4f8c46a RBX: ffff880045683a70 RCX:
0000000000000000
<4>[1184369.677630] RDX: 000000000000058d RSI: 0000000000000001 RDI:
ffff880045683af8
<4>[1184369.678152] RBP: ffff880045683a60 R08: 0000000000000400 R09:
000000000000000c
<4>[1184369.678673] R10: ffff8807e4f8c462 R11: 0000000000000000 R12:
ffff8807f8c5c4c0
<4>[1184369.679194] R13: ffffffffa01ea620 R14: ffff8807e4f8c462 R15:
ffff8807e4f8c46a
<4>[1184369.679715] FS: 0000000000000000(0000) GS:ffff880045680000(0000)
knlGS:0000000000000000
<4>[1184369.680233] CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b
<4>[1184369.680495] CR2: 00007f88776d6000 CR3: 00000008263db000 CR4:
00000000001406e0
<4>[1184369.681009] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
0000000000000000
<4>[1184369.681522] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7:
0000000000000400
<4>[1184369.682037] Process swapper (pid: 0, threadinfo ffff880833e76000, task
ffff880833e4a040)
<4>[1184369.682558] Stack:
<4>[1184369.682819] 0000000039000400 ffff88082dbd0118 ffff880045683aa0
ffffffffa01d7b64
<4>[1184369.683092] <d> 0008000002000000 ffff8807e4f8c478 ffff88082dbf8c69
ffffffffa0077d38
<4>[1184369.683627] <d> 0000000000000001 ffff88082edec800 0000000100000000
ffff8807f8c5c4c0
<4>[1184369.684421] Call Trace:
<4>[1184369.684682] <IRQ>
<4>[1184369.685470] [<ffffffffa01d96c4>]
ovs_dp_process_received_packet+0x44/0x80 [openvswitch]
<4>[1184369.685994] [<ffffffffa01e115e>] ovs_vport_receive+0x2e/0x30
[openvswitch]
<4>[1184369.686261] [<ffffffffa01e20cc>] gre_rcv+0xac/0xd0 [openvswitch]
<4>[1184369.686529] [<ffffffffa01e5442>] gre_cisco_rcv+0x1c2/0x310
[openvswitch]
<4>[1184369.686797] [<ffffffffa01e5259>] gre_rcv+0x59/0x80 [openvswitch]
<4>[1184369.687063] [<ffffffff8146197d>] ip_local_deliver_finish+0xdd/0x2d0
<4>[1184369.687331] [<ffffffff81461c08>] ip_local_deliver+0x98/0xa0
<4>[1184369.687596] [<ffffffff814610cd>] ip_rcv_finish+0x12d/0x440
<4>[1184369.687861] [<ffffffff81461655>] ip_rcv+0x275/0x350
<4>[1184369.694516] [<ffffffff8142bc6b>] __netif_receive_skb+0x49b/0x6f0
<4>[1184369.694779] [<ffffffff8142dde8>] netif_receive_skb+0x58/0x60
<4>[1184369.695041] [<ffffffff8142def0>] napi_skb_finish+0x50/0x70
<4>[1184369.695303] [<ffffffff8142ff09>] napi_gro_receive+0x39/0x50
<4>[1184369.695568] [<ffffffffa0148a02>] ixgbe_receive_skb+0x52/0x90 [ixgbe]
<4>[1184369.695832] [<ffffffffa014b485>] ixgbe_poll+0x445/0x1760 [ixgbe]
<4>[1184369.696100] [<ffffffff811043ab>] ? perf_pmu_enable+0x2b/0x40
<4>[1184369.696366] [<ffffffff8110a0b8>] ? perf_event_task_tick+0xa8/0x2f0
<4>[1184369.696631] [<ffffffff81430023>] net_rx_action+0x103/0x2f0
<4>[1184369.696899] [<ffffffff81072911>] __do_softirq+0xc1/0x1e0
<4>[1184369.697165] [<ffffffff810d8ae0>] ? handle_IRQ_event+0x60/0x170
<4>[1184369.697430] [<ffffffff8100c1ec>] call_softirq+0x1c/0x30
<4>[1184369.697694] [<ffffffff8100de85>] do_softirq+0x65/0xa0
<4>[1184369.697956] [<ffffffff810726f5>] irq_exit+0x85/0x90
<4>[1184369.698221] [<ffffffff814f4a25>] do_IRQ+0x75/0xf0
<4>[1184369.698484] [<ffffffff8100b9d3>] ret_from_intr+0x0/0x11
It crashed in ovs_flow_extract with _skb_pull the src&dst mac address
(BUG_ON(skb->len < skb->data_len);)
int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key)
{
.....
__skb_pull(skb, 2 * ETH_ALEN);
.....
}
I get the skb pointer: It's true len is less than data_len and protocol is
0x2558
crash> struct sk_buff ffff8807f8c5c4c0
struct sk_buff {
next = 0x0,
prev = 0x0,
sk = 0x0,
....
len = 1421,
data_len = 1429,
....
protocol = 22565,
....
transport_header = 98,
network_header = 78,
mac_header = 106,
tail = 110,
end = 640,
head = 0xffff8807e4f8c400 "",
data = 0xffff8807e4f8c46a "RS\001",
truesize = 3328,
users = {
counter = 1
}
}
Get the iphdr, it is a gre packet
crash> struct iphdr ffff8807e4f8c44e
struct iphdr {
ihl = 5 '\005',
version = 4 '\004',
tos = 0 '\000',
tot_len = 46341,
id = 12135,
frag_off = 64,
ttl = 57 '9',
protocol = 47 '/',
check = 30992,
saddr = 1046812588,
daddr = 34020012
}
But I find the gre header get some mess, gre->protocol is not
ETH_P_TEB(0x6558). but 22565 convert to host-edian is 0x2558
I think the packet get some error in the air, But I think ovs should not lead
kernel crash
crash> struct gre_base_hdr ffff8807e4f8c462
struct gre_base_hdr {
flags = 32,
protocol = 22565
}
I think most part of the packet is in the nolinear-spatial.
1. gre_rcv: pskb_may_pull(skb, 12)
pull the 12 bytes to linear-spatial(skb->data). The gre header is 8 bytes with
key.
2. gre_cisco_rcv-->parse_gre_header-->iptunnel_pull_header
{
if (inner_proto == htons(ETH_P_TEB)) {
struct ethhdr *eh;
if (unlikely(!pskb_may_pull(skb, ETH_HLEN)))
return -ENOMEM;
....
}
else
skb->protocol = inner_proto;
}
So the wrong inner_proto leads no pull the Mac header to
linear-spatial(skb->data)
3. finally It made a crash in ovs_flow_extract->_pull_skb
I think ovs should check this mess situation in two ways.
1. check the tpi->proto
static int gre_rcv(struct sk_buff *skb,
const struct tnl_ptk_info *tpi)
{
struct ovs_key_ipv4_tunnel tun_key;
struct ovs_net *ovs_net;
struct vport *vport;
__be64 key;
ovs_net = net_generic(dev_net(skb->dev), ovs_net_id);
if ((tpi->flags & TUNNEL_KEY) && (tpi->flags & TUNNEL_SEQ))
vport = rcu_dereference(ovs_net->vport_net.gre64_vport);
else
vport = rcu_dereference(ovs_net->vport_net.gre_vport);
if (unlikely(!vport))
return PACKET_REJECT;
+ if (unlikely(tpi->proto != htons(ETH_P_TEB)))
+ return PACKET_REJECT;
key = key_to_tunnel_id(tpi->key, tpi->seq);
ovs_flow_tun_key_init(&tun_key, ip_hdr(skb), key,
filter_tnl_flags(tpi->flags));
ovs_vport_receive(vport, skb, &tun_key);
return PACKET_RCVD;
}
2. add pskb_may_pull before pull like ip_gre did
int ovs_flow_extract(struct sk_buff *skb, u16 in_port, struct sw_flow_key *key)
{
+ if (unlikely(!pskb_may_pull(skb, ETH_HLEN)))
+ return -ENOMEM;
__skb_pull(skb, 2 * ETH_ALEN);
}
_______________________________________________
discuss mailing list
[email protected]
http://openvswitch.org/mailman/listinfo/discuss
