On 24 Oct 2023, at 8:47, chuanyun Xiao wrote:
> I use the VM to ping an external IP address and telnet same ip To observe
> the tc rules of bond0。
>
> [root@gateway02 ~]# tc filter show egress dev bond0
> filter block 18 protocol ip pref 2 flower chain 0
> filter block 18 protocol ip pref 2 flower chain 0 handle 0x1
> dst_mac e8:eb:d3:3a:d5:e8
> src_mac 88:2a:5e:a9:30:ad
> eth_type ipv4
> ip_flags nofrag
> in_hw in_hw_count 2
> action order 1: skbedit ptype host pipe
> index 1 ref 1 bind 1
> used_hw_stats delayed
>
> action order 2: mirred (Ingress Redirect to device br-ex) stolen
> index 2 ref 1 bind 1
> cookie 269afd7f074e8f0f9376cf88d44bb2e7
> no_percpu
> used_hw_stats delayed
>
> filter block 18 protocol ip pref 2 flower chain 0 handle 0x2
> dst_mac ae:86:86:ab:a6:08
> src_mac 88:2a:5e:a9:30:ad
> eth_type ipv4
> ip_proto tcp
> ip_tos 0/0x3
> ip_ttl 58
> dst_ip 10.199.100.20
> src_ip 10.79.254.16/9
> ip_flags nofrag
> ct_state -trk-inv
> in_hw in_hw_count 2
> action order 1: tunnel_key set
> src_ip 0.0.0.0
> dst_ip 172.31.2.22
> key_id 5
> dst_port 6081
> geneve_opts 0102:80:00010005
> csum
> ttl 64 pipe
> index 2 ref 1 bind 1
> no_percpu
> used_hw_stats delayed
>
> action order 2: pedit action pipe keys 4
> index 3 ref 1 bind 1
> key #0 at eth+4: val 0000aebf mask ffff0000
> key #1 at eth+8: val bf9ebc20 mask 00000000
> key #2 at eth+0: val 00001991 mask 00000000
> key #3 at eth+4: val 00200000 mask 0000ffff
> used_hw_stats delayed
>
> action order 3: pedit action pipe keys 1
> index 4 ref 1 bind 1
> key #0 at ipv4+8: val 38000000 mask 00ffffff
> used_hw_stats delayed
>
> action order 4: csum (iph, tcp) action pipe
> index 2 ref 1 bind 1
> no_percpu
> used_hw_stats delayed
>
> action order 5: mirred (Egress Redirect to device genev_sys_6081)
> stolen
> index 4 ref 1 bind 1
> cookie 65a40a0b9e4d542b353344b28762f37d
> no_percpu
> used_hw_stats delayed
>
> filter block 18 protocol ip pref 2 flower chain 0 handle 0x3
> dst_mac ae:86:86:ab:a6:08
> src_mac 88:2a:5e:a9:30:ad
> eth_type ipv4
> ip_proto icmp
> ip_tos 0/0x3
> ip_ttl 58
> dst_ip 10.199.100.20
> src_ip 10.79.254.16/9
> icmp_type 0/254
> ip_flags nofrag
> ct_state -trk-inv
> not_in_hw
> action order 1: tunnel_key set
> src_ip 0.0.0.0
> dst_ip 172.31.2.22
> key_id 5
> dst_port 6081
> geneve_opts 0102:80:00010005
> csum
> ttl 64 pipe
> index 4 ref 1 bind 1
> no_percpu
>
> action order 2: pedit action pipe keys 4
> index 7 ref 1 bind 1
> key #0 at eth+4: val 0000aebf mask ffff0000
> key #1 at eth+8: val bf9ebc20 mask 00000000
> key #2 at eth+0: val 00001991 mask 00000000
> key #3 at eth+4: val 00200000 mask 0000ffff
>
> action order 3: pedit action pipe keys 1
> index 8 ref 1 bind 1
> key #0 at ipv4+8: val 38000000 mask 00ffffff
>
> action order 4: csum (iph) action pipe
> index 4 ref 1 bind 1
> no_percpu
>
> action order 5: mirred (Egress Redirect to device genev_sys_6081)
> stolen
> index 6 ref 1 bind 1
> cookie bf012388974ae42e806cbd87282c5b92
> no_percpu
>
>
> I use ConnectX-6 Dx OFED：5.9-0.5.5 firmware-version: 22.38.1002
> (MT_0000000359)
>
>
> Can you help me find out what is the difference between these two tc rules
> that causes ICMP to fail to hardware offload? What is the final solution？
> I can provide more information if needed，Thank you!
I do not see anything that stands out other than using ICMP vs TCP. I guess
your next step would be to ask Nvidia why their CX6 is not offloading this flow.
//Eelco
>
> Eelco Chaudron <echau...@redhat.com> 于2023年10月24日周二 14:34写道：
>
>>
>>
>> On 24 Oct 2023, at 7:02, chuanyun Xiao via discuss wrote:
>>
>>> I use the VM to ping an external IP address and view tc rules on the
>>> centralized gateway
>>>
>>> filter block 18 protocol ip pref 2 flower chain 0 handle 0x2
>>> dst_mac ae:86:86:ab:a6:08
>>> src_mac 88:2a:5e:a9:30:ad
>>> eth_type ipv4
>>> ip_proto icmp
>>> ip_tos 0/0x3
>>> ip_ttl 58
>>> dst_ip 10.199.100.20
>>> src_ip 10.79.254.16/9
>>> icmp_type 0/254
>>> ip_flags nofrag
>>> ct_state -trk-inv
>>> not_in_hw
>>> action order 1: tunnel_key set
>>> src_ip 0.0.0.0
>>> dst_ip 172.31.2.22
>>> key_id 5
>>> dst_port 6081
>>> geneve_opts 0102:80:00010005
>>> csum
>>> ttl 64 pipe
>>> index 2 ref 1 bind 1
>>> no_percpu
>>>
>>> action order 2: pedit action pipe keys 4
>>> index 3 ref 1 bind 1
>>> key #0 at eth+4: val 0000aebf mask ffff0000
>>> key #1 at eth+8: val bf9ebc20 mask 00000000
>>> key #2 at eth+0: val 00001991 mask 00000000
>>> key #3 at eth+4: val 00200000 mask 0000ffff
>>>
>>> action order 3: pedit action pipe keys 1
>>> index 4 ref 1 bind 1
>>> key #0 at ipv4+8: val 38000000 mask 00ffffff
>>>
>>> action order 4: csum (iph) action pipe
>>> index 2 ref 1 bind 1
>>> no_percpu
>>>
>>> action order 5: mirred (Egress Redirect to device genev_sys_6081)
>>> stolen
>>> index 4 ref 1 bind 1
>>> cookie bf012388974ae42e806cbd87282c5b92
>>> no_percpu
>>>
>>> The TC rule is not_in_hw。 ip_proto TCP or UDP is in_hw。
>>>
>>>
>>> [root@gateway02 ~]# ovs-appctl dpctl/dump-flows -m
>>> ufid:74a18009-31cd-4d5e-b958-4614cfc09409,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.2.22,dst=172.31.7.24,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10005/0x7fffffff}),flags(+key)),in_port(genev_sys_6081),skb_mark(0/0),ct_state(0/0x30),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=ae:68:68:78:2c:8a,dst=ae:1e:1e:fe:14:5f),eth_type(0x0800),ipv4(src=
>>>
>> 10.199.100.16/255.255.255.240,dst=10.0.0.0/255.128.0.0,proto=1,tos=0/0,ttl=63,frag=no
>> ),icmp(type=8/0xf8,code=0/0),
>>> packets:0, bytes:0, used:never, dp:tc,
>>>
>> actions:set(eth(src=ae:86:86:ab:a6:08,dst=88:2a:5e:a9:30:ad)),set(ipv4(ttl=62)),bond0
>>> ufid:81f41145-4897-4f0a-bc53-3117f646b4cc,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),tunnel(tun_id=0x1,src=172.31.2.22,dst=172.31.7.24,ttl=0/0,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10005/0x7fffffff}),flags(+key)),in_port(genev_sys_6081),skb_mark(0/0),ct_state(0/0x30),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=ae:68:68:78:2c:8a,dst=ae:1e:1e:fe:14:5f),eth_type(0x0800),ipv4(src=
>>>
>> 10.199.100.16/255.255.255.240,dst=10.0.0.0/255.128.0.0,proto=6,tos=0/0,ttl=63,frag=no
>> ),tcp(src=0/0,dst=0/0),
>>> packets:2, bytes:132, used:1.110s, offloaded:yes, dp:tc,
>>>
>> actions:set(eth(src=ae:86:86:ab:a6:08,dst=88:2a:5e:a9:30:ad)),set(ipv4(ttl=62)),bond0
>>> ufid:7ffd9a26-0f8f-4e07-88cf-7693e7b24bd4,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(bond0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ad,dst=e8:eb:d3:3a:d5:e8),eth_type(0x0800),ipv4(src=
>>> 0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no),
>>> packets:133315, bytes:12479861, used:1.110s, offloaded:yes, dp:tc,
>>> actions:br-ex
>>> ufid:882301bf-2ee4-4a97-87bd-6c80925b2c28,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(bond0),skb_mark(0/0),ct_state(0/0x30),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ad,dst=ae:86:86:ab:a6:08),eth_type(0x0800),ipv4(src=
>>> 10.0.0.0/255.128.0.0,dst=10.199.100.20,proto=1,tos=0/0x3,ttl=58,frag=no
>> ),icmp(type=0/0xfe,code=0/0),
>>> packets:0, bytes:0, used:never, dp:tc,
>>>
>> actions:set(tunnel(tun_id=0x5,dst=172.31.2.22,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10005}),flags(csum|key))),set(eth(src=ae:bf:bf:9e:bc:20,dst=00:00:19:91:00:20)),set(ipv4(ttl=56)),genev_sys_6081
>>> ufid:02e435a6-c803-4a06-b97b-3dce5f36c633,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(bond0),skb_mark(0/0),ct_state(0/0x30),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:ad,dst=ae:86:86:ab:a6:08),eth_type(0x0800),ipv4(src=
>>> 10.0.0.0/255.128.0.0,dst=10.199.100.20,proto=6,tos=0/0x3,ttl=58,frag=no
>> ),tcp(src=0/0,dst=0/0),
>>> packets:2, bytes:140, used:1.110s, offloaded:yes, dp:tc,
>>>
>> actions:set(tunnel(tun_id=0x5,dst=172.31.2.22,ttl=64,tp_dst=6081,geneve({class=0x102,type=0x80,len=4,0x10005}),flags(csum|key))),set(eth(src=ae:bf:bf:9e:bc:20,dst=00:00:19:91:00:20)),set(ipv4(ttl=56)),genev_sys_6081
>>> ufid:ba6a5e73-ca18-43fb-b133-ec96520a329b,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(bond0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=88:2a:5e:a9:30:f6,dst=01:80:c2:00:00:0e),eth_type(0x88cc),
>>> packets:0, bytes:0, used:never, offloaded:yes, dp:tc, actions:drop
>>> ufid:40df4ad6-ba18-4589-a9c3-98b2ba4ad384,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(br-ex),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),packet_type(ns=0/0,id=0/0),eth(src=e8:eb:d3:3a:d5:e8,dst=88:2a:5e:a9:30:ad),eth_type(0x0800),ipv4(src=
>>> 0.0.0.0/0.0.0.0,dst=0.0.0.0/0.0.0.0,proto=0/0,tos=0/0,ttl=0/0,frag=no),
>>> packets:72860, bytes:7469155, used:1.110s, offloaded:yes, dp:tc,
>>> actions:bond0
>>> ufid:bc1182f6-0cd0-4d2d-9eea-8a4df1ab3cc3,
>>>
>> recirc_id(0),dp_hash(0/0),skb_priority(0/0),in_port(bond0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),eth(src=88:2a:5e:a9:30:f6,dst=01:80:c2:00:00:00),eth_type(0/0xffff),
>>> packets:161022, bytes:19161618, used:1.934s, dp:ovs, actions:drop
>>>
>>>
>>> system is a AlmaLinux release 8.7 with kernel 4.18.0, OVS 3.2.0 and OVN
>>> 23.03.1.
>>>
>>> How can I make ICMP flow hardware offload？
>>
>> As the rule is in TC it’s the NIC driver deciding if the flow can be
>> offloaded or not. If you compare the tc rule for TCP and ICMP is there any
>> difference that stands out that could permit the hardware from offloading
>> it?
>>
>> //Eelco
>>
>>> _______________________________________________
>>> discuss mailing list
>>> disc...@openvswitch.org
>>> https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
>>
>>
_______________________________________________
discuss mailing list
disc...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
