Hi all, I'm using ovs-dpdk(ovs 2.17.2, dpdk 21.11.1).
Now I found a bug: Why snat action NOT work while using non-DPDK port in openflow. 1. how to reproduce ``` ### add bridge and port link this [root@bogon ~]# ovs-vsctl show 272472f6-3ab9-4cd5-b1dd-76a63f0e6127 Bridge brp0 datapath_type: netdev Port pf0hpf Interface pf0hpf type: dpdk options: {dpdk-devargs="0000:01:00.0,representor=[65535]"} Port brp0 Interface brp0 type: internal Port brp0-patch Interface brp0-patch type: patch options: {peer=br-int-patch} Port p0 Interface p0 type: dpdk options: {dpdk-devargs="0000:01:00.0"} Bridge br-int datapath_type: netdev Port s0 Interface s0 Port br-int-patch Interface br-int-patch type: patch options: {peer=brp0-patch} Port pf0vf2 Interface pf0vf2 type: dpdk options: {dpdk-devargs="0000:01:00.0,representor=[2]"} Port pf0vf0 Interface pf0vf0 type: dpdk options: {dpdk-devargs="0000:01:00.0,representor=[0]"} Port pf0vf1 Interface pf0vf1 type: dpdk options: {dpdk-devargs="0000:01:00.0,representor=[1]"} Port geneve1 Interface geneve1 type: geneve options: {key=flow, remote_ip="10.0.0.2"} Port pf0vf3 Interface pf0vf3 type: dpdk options: {dpdk-devargs="0000:01:00.0,representor=[3]"} Port br-int Interface br-int type: internal ovs_version: "2.17.2" [root@bogon ~]# [root@bogon ~]# ovs-ofctl dump-flows br-int cookie=0x0, duration=4908.200s, table=0, n_packets=25, n_bytes=1898, priority=20,ip,metadata=0x1,in_port=s0,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:02:04,resubmit(,1) cookie=0x0, duration=4908.178s, table=0, n_packets=25, n_bytes=1898, priority=30,ip,metadata=0x2,in_port=s0,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:04:01,resubmit(,1) cookie=0x0, duration=4908.141s, table=0, n_packets=2, n_bytes=196, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.0.0.3 actions=ct(table=1,nat) cookie=0x0, duration=4908.111s, table=0, n_packets=2, n_bytes=196, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.100.0.2 actions=dec_ttl,mod_dl_src:00:00:00:33:00:01,resubmit(,1) cookie=0x0, duration=4908.368s, table=0, n_packets=7, n_bytes=294, priority=10,arp,in_port=s0 actions=resubmit(,1) cookie=0x0, duration=4908.271s, table=0, n_packets=1, n_bytes=60, priority=10,arp,in_port="br-int-patch" actions=resubmit(,1) cookie=0x0, duration=4908.206s, table=0, n_packets=25, n_bytes=1898, priority=10,ip,in_port=s0,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2 actions=clone(load:0x1->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=4908.148s, table=0, n_packets=2, n_bytes=196, priority=10,ip,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.0.0.3 actions=clone(load:0x2->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=4908.193s, table=1, n_packets=25, n_bytes=1898, priority=20,ip,metadata=0x1,in_port=s0,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2 actions=mod_dl_dst:00:00:00:00:02:05,resubmit(,2) cookie=0x0, duration=4908.170s, table=1, n_packets=25, n_bytes=1898, priority=30,ip,metadata=0x2,in_port=s0,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=mod_dl_dst:00:00:00:00:03:01,resubmit(,2) cookie=0x0, duration=4908.133s, table=1, n_packets=2, n_bytes=196, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.100.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:02:05,resubmit(,2) cookie=0x0, duration=4908.103s, table=1, n_packets=2, n_bytes=196, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.100.0.2 actions=mod_dl_dst:00:00:00:33:00:02,resubmit(,2) cookie=0x0, duration=4908.337s, table=1, n_packets=7, n_bytes=294, priority=10,arp,in_port=s0,arp_tpa=10.100.0.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:00:00:00:33:00:01,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x330001->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xa640001->NXM_OF_ARP_SPA[],resubmit(,2) cookie=0x0, duration=4908.240s, table=1, n_packets=1, n_bytes=60, priority=10,arp,in_port="br-int-patch",arp_tpa=10.0.0.3,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:00:00:00:00:04:01,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x401->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xa000003->NXM_OF_ARP_SPA[],resubmit(,2) cookie=0x0, duration=4908.186s, table=2, n_packets=25, n_bytes=1898, priority=20,ip,metadata=0x1,in_port=s0,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=clone(ct_clear,load:0x2->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=4908.163s, table=2, n_packets=25, n_bytes=1898, priority=30,ip,metadata=0x2,in_port=s0,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2 actions=ct(commit,table=3,nat(src=10.0.0.3)) cookie=0x0, duration=4908.125s, table=2, n_packets=2, n_bytes=196, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.100.0.2 actions=mod_dl_dst:00:00:00:00:02:04,resubmit(,3) cookie=0x0, duration=4908.095s, table=2, n_packets=2, n_bytes=196, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:33:00:02,nw_dst=10.100.0.2 actions=clone(ct_clear,output:s0) cookie=0x0, duration=4908.306s, table=2, n_packets=7, n_bytes=294, priority=10,arp,arp_spa=10.100.0.1 actions=IN_PORT cookie=0x0, duration=4908.210s, table=2, n_packets=1, n_bytes=60, priority=10,arp,arp_spa=10.0.0.3 actions=IN_PORT cookie=0x0, duration=4908.155s, table=3, n_packets=25, n_bytes=1898, priority=30,ip,metadata=0x2,in_port=s0,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2 actions=clone(ct_clear,output:"br-int-patch") cookie=0x0, duration=4908.118s, table=3, n_packets=2, n_bytes=196, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.100.0.2 actions=clone(ct_clear,load:0x1->OXM_OF_METADATA[],resubmit(,0)) [root@bogon ~]# ovs-ofctl dump-flows brp0 cookie=0x0, duration=4911.174s, table=0, n_packets=26, n_bytes=1958, in_port="brp0-patch" actions=output:p0 cookie=0x0, duration=4911.166s, table=0, n_packets=6, n_bytes=466, in_port=p0 actions=output:"brp0-patch" ### send packet from s0 port [root@bogon ~]# ip netns exec nss0 ip a 1: lo: <LOOPBACK> mtu 65536 qdisc noop state DOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2: vrf-blue: <NOARP,MASTER,UP,LOWER_UP> mtu 65575 qdisc noqueue state UP group default qlen 1000 link/ether 46:ad:11:74:b5:1d brd ff:ff:ff:ff:ff:ff 252: s0p@if253: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 link/ether 00:00:00:33:00:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0 inet 10.100.0.2/24 scope global s0p valid_lft forever preferred_lft forever inet6 fe80::200:ff:fe33:2/64 scope link dadfailed tentative valid_lft forever preferred_lft forever [root@bogon ~]# ip netns exec nss0 iperf3 -b 1M -c 10.0.0.2 -t 30 ### check packet out and megaflow, found ERROR out packet's ipv4 src is 10.100.0.2, why it's NOT 10.0.0.3? Which means snat action NOT work. the megaflow is: [root@bogon ~]# ovs-appctl dpctl/dump-flows -m flow-dump from the main thread: ufid:068f7950-26e2-4532-a3fa-02918758c8bb, skb_priority(0/0),skb_mark(0/0),ct_state(0/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0),dp_hash(0/0),in_port(s0),packet_type(ns=0,id=0),eth(src=00:00:00:33:00:02,dst=00:00:00:33:00:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,tos=0/0,ttl=64,frag=no),tcp(src=42448,dst=5201),tcp_flags(0/0), packets:1, bytes:74, offload_packets:0, offload_bytes:0, used:0.068s, flags:S, dp:ovs, actions:set(eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01)),set(ipv4(ttl=62)),ct(commit,nat(src=10.0.0.3)),recirc(0x2a), dp-extra-info:miniflow_bits(5,3) ufid:b145dd3c-0c37-46c3-a912-9c2024a63bd4, skb_priority(0/0),skb_mark(0/0),ct_state(0x30/0),ct_zone(0/0),ct_mark(0/0),ct_label(0/0),recirc_id(0x2a),dp_hash(0/0),in_port(s0),packet_type(ns=0,id=0),eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,tos=0/0,ttl=62/0,frag=no),tcp(src=42448,dst=5201),tcp_flags(0/0), packets:1, bytes:74, offload_packets:0, offload_bytes:0, used:0.068s, flags:S, dp:ovs, actions:ct_clear,p0, dp-extra-info:miniflow_bits(5,3) ### check log 2024-06-14T06:39:07.562Z|00310|dpif_netdev|DBG|ovs-netdev: miss upcall: skb_priority(0),skb_mark(0),ct_state(0),ct_zone(0),ct_mark(0),ct_label(0),recirc_id(0),dp_hash(0),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:33:00:02,dst=00:00:00:33:00:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,tos=0,ttl=64,frag=no),tcp(src=60170,dst=5201),tcp_flags(syn) tcp,vlan_tci=0x0000,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=60170,tp_dst=5201,tcp_flags=syn tcp_csum:1496 2024-06-14T06:39:07.562Z|00312|dpif_netdev|DBG|Creating 1. subtable 0x1cba550 for in_port 10 2024-06-14T06:39:07.562Z|00313|dpif_netdev|DBG|netdev_flow ufid:2bbc60f3-6b57-4927-b298-bf77a29d99d1 actions cannot offload: set(eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01)),set(ipv4(ttl=62)),ct(commit,nat(src=10.0.0.3)),recirc(0x2c) 2024-06-14T06:39:07.562Z|00314|dpif_netdev|DBG|flow_add: ufid:2bbc60f3-6b57-4927-b298-bf77a29d99d1 mega_ufid:2b93ee2b-7548-4680-81f3-30d47c6c8dd7 recirc_id(0),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:33:00:02,dst=00:00:00:33:00:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,ttl=64,frag=no),tcp(src=60170,dst=5201), actions:set(eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01)),set(ipv4(ttl=62)),ct(commit,nat(src=10.0.0.3)),recirc(0x2c) 2024-06-14T06:39:07.562Z|00315|dpif_netdev|DBG|flow match: recirc_id=0,eth,tcp,vlan_tci=0x0000/0x1fff,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_ttl=64,nw_frag=no,tp_src=60170,tp_dst=5201 2024-06-14T06:39:07.562Z|00316|dpif_netdev|DBG|ovs-netdev: miss upcall: skb_priority(0),skb_mark(0),ct_state(0x30),ct_zone(0),ct_mark(0),ct_label(0),recirc_id(0x2c),dp_hash(0),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,tos=0,ttl=62,frag=no),tcp(src=60170,dst=5201),tcp_flags(syn) tcp,vlan_tci=0x0000,dl_src=00:00:00:00:04:01,dl_dst=00:00:00:00:03:01,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=62,tp_src=60170,tp_dst=5201,tcp_flags=syn tcp_csum:1496 2024-06-14T06:39:07.562Z|00318|dpif_netdev|DBG|Creating 2. subtable 0x1d00a80 for in_port 10 2024-06-14T06:39:07.563Z|00319|dpif_netdev|DBG|flow_add: ufid:012ac7cc-091d-491c-a597-6c3732b10fdb mega_ufid:b339b485-0c64-4eee-85be-636dcd63a97f recirc_id(0x2c),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01),eth_type(0x0800),ipv4(src=10.100.0.2,dst=10.0.0.2,proto=6,frag=no),tcp(src=60170,dst=5201), actions:ct_clear,7 2024-06-14T06:39:07.563Z|00320|dpif_netdev|DBG|flow match: recirc_id=0x2c,eth,tcp,vlan_tci=0x0000/0x1fff,dl_src=00:00:00:00:04:01,dl_dst=00:00:00:00:03:01,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_frag=no,tp_src=60170,tp_dst=5201 2024-06-14T06:39:07.608Z|00001|dpif_netdev(hw_offload5)|DBG|Mark id for ufid b339b485-0c64-4eee-85be-636dcd63a97f was not found 2024-06-14T06:39:07.608Z|00002|dpif_netdev(hw_offload5)|DBG|failed to add netdev flow b339b485-0c64-4eee-85be-636dcd63a97f 2024-06-14T06:39:12.580Z|00321|dpif_netdev|DBG|ovs-netdev: miss upcall: skb_priority(0),skb_mark(0),ct_state(0),ct_zone(0),ct_mark(0),ct_label(0),recirc_id(0),dp_hash(0),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:33:00:02,dst=00:00:00:33:00:01),eth_type(0x0806),arp(sip=10.100.0.2,tip=10.100.0.1,op=1,sha=00:00:00:33:00:02,tha=00:00:00:00:00:00) arp,vlan_tci=0x0000,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,arp_spa=10.100.0.2,arp_tpa=10.100.0.1,arp_op=1,arp_sha=00:00:00:33:00:02,arp_tha=00:00:00:00:00:00 2024-06-14T06:39:12.580Z|00322|flow|DBG|Flow is not IPv4/IPv6. 2024-06-14T06:39:12.580Z|00323|flow|DBG|Flow can't be offloaded due to missing mandatory fields required by hardware. 2024-06-14T06:39:12.580Z|00324|dpif_netdev|DBG|Creating 3. subtable 0x1d01820 for in_port 10 2024-06-14T06:39:12.580Z|00325|dpif_netdev|DBG|flow_add: ysk2_flag:0 ufid:77d38ebe-0921-437a-8af5-da906a7e0ef9 mega_ufid:77d38ebe-0921-437a-8af5-da906a7e0ef9 recirc_id(0),in_port(10),packet_type(ns=0,id=0),eth(src=00:00:00:33:00:02,dst=00:00:00:33:00:01),eth_type(0x0806),arp(sip=10.100.0.2,tip=10.100.0.1,op=1,sha=00:00:00:33:00:02,tha=00:00:00:00:00:00), actions:userspace(pid=0,slow_path(action)) 2024-06-14T06:39:12.580Z|00326|dpif_netdev|DBG|flow match: recirc_id=0,eth,arp,vlan_tci=0x0000/0x1fff,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,arp_spa=10.100.0.2,arp_tpa=10.100.0.1,arp_op=1,arp_sha=00:00:00:33:00:02,arp_tha=00:00:00:00:00:00 2024-06-14T06:39:12.580Z|00327|dpif_netdev|DBG|Core -1: New TX queue ID 0 for port 's0'. ``` 2. I debug using this: ``` [root@bogon ~]# ovs-appctl ofproto/trace br-int in_port=s0,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,tcp,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_ttl=64 Flow: tcp,in_port=7,vlan_tci=0x0000,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,nw_src=10.100.0.2,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=0,tcp_flags=0 bridge("br-int") ---------------- 0. ip,in_port=7,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2, priority 10 clone(load:0x1->OXM_OF_METADATA[],resubmit(,0)) load:0x1->OXM_OF_METADATA[] resubmit(,0) 0. ip,metadata=0x1,in_port=7,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2, priority 20 dec_ttl mod_dl_src:00:00:00:00:02:04 resubmit(,1) 1. ip,metadata=0x1,in_port=7,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2, priority 20 mod_dl_dst:00:00:00:00:02:05 resubmit(,2) 2. ip,metadata=0x1,in_port=7,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 20 clone(ct_clear,load:0x2->OXM_OF_METADATA[],resubmit(,0)) ct_clear load:0x2->OXM_OF_METADATA[] resubmit(,0) 0. ip,metadata=0x2,in_port=7,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 30 dec_ttl mod_dl_src:00:00:00:00:04:01 resubmit(,1) 1. ip,metadata=0x2,in_port=7,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 30 mod_dl_dst:00:00:00:00:03:01 resubmit(,2) 2. ip,metadata=0x2,in_port=7,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2, priority 30 ct(commit,table=3,nat(src=10.0.0.3)) nat(src=10.0.0.3) -> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 3. -> Sets the packet to an untracked state, and clears all the conntrack fields. Final flow: unchanged Megaflow: recirc_id=0,eth,ip,in_port=7,dl_src=00:00:00:33:00:02,dl_dst=00:00:00:33:00:01,nw_dst=10.0.0.2,nw_ttl=64,nw_frag=no Datapath actions: set(eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01)),set(ipv4(ttl=62)),ct(commit,nat(src=10.0.0.3)),recirc(0x29) =============================================================================== recirc(0x29) - resume conntrack with default ct_state=trk|new (use --ct-next to customize) Replacing src/dst IP/ports to simulate NAT: Initial flow: nw_src=10.100.0.2,tp_src=0,nw_dst=10.0.0.2,tp_dst=0 Modified flow: nw_src=10.0.0.3,tp_src=0,nw_dst=10.0.0.2,tp_dst=0 =============================================================================== Flow: recirc_id=0x29,ct_state=new|trk,eth,tcp,metadata=0x2,in_port=7,vlan_tci=0x0000,dl_src=00:00:00:00:04:01,dl_dst=00:00:00:00:03:01,nw_src=10.0.0.3,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=62,tp_src=0,tp_dst=0,tcp_flags=0 bridge("br-int") ---------------- thaw Resuming from table 3 3. ip,metadata=0x2,in_port=7,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2, priority 30 clone(ct_clear,output:5) ct_clear output:5 bridge("brp0") -------------- 0. in_port=3, priority 32768 output:1 Final flow: unchanged Megaflow: recirc_id=0x29,eth,ip,in_port=7,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2,nw_frag=no Datapath actions: ct_clear,7 ``` 3. Notice, as compare, if I use the port 'pf0vf0', which is DPDK type port, there is NO bug. Which means pmd thread process this case is OK. ``` ### NOTICE, for debug using 'pf0vf0', I change openflow, NOT same as above [root@bogon ~]# ovs-ofctl dump-flows br-int cookie=0x0, duration=69.699s, table=0, n_packets=830, n_bytes=1234642, priority=20,ip,metadata=0x1,in_port=pf0vf2,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:02:04,resubmit(,1) cookie=0x0, duration=69.677s, table=0, n_packets=830, n_bytes=1234642, priority=30,ip,metadata=0x2,in_port=pf0vf2,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:04:01,resubmit(,1) cookie=0x0, duration=69.639s, table=0, n_packets=340, n_bytes=22460, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.0.0.3 actions=ct(table=1,nat) cookie=0x0, duration=69.609s, table=0, n_packets=340, n_bytes=22460, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.2.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:02:02,resubmit(,1) cookie=0x0, duration=69.874s, table=0, n_packets=0, n_bytes=0, priority=10,arp,in_port=pf0vf2 actions=resubmit(,1) cookie=0x0, duration=69.777s, table=0, n_packets=1, n_bytes=60, priority=10,arp,in_port="br-int-patch" actions=resubmit(,1) cookie=0x0, duration=69.707s, table=0, n_packets=830, n_bytes=1234642, priority=10,ip,in_port=pf0vf2,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2 actions=clone(load:0x1->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=69.647s, table=0, n_packets=340, n_bytes=22460, priority=10,ip,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.0.0.3 actions=clone(load:0x2->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=69.692s, table=1, n_packets=830, n_bytes=1234642, priority=20,ip,metadata=0x1,in_port=pf0vf2,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2 actions=mod_dl_dst:00:00:00:00:02:05,resubmit(,2) cookie=0x0, duration=69.669s, table=1, n_packets=830, n_bytes=1234642, priority=30,ip,metadata=0x2,in_port=pf0vf2,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=mod_dl_dst:00:00:00:00:03:01,resubmit(,2) cookie=0x0, duration=69.631s, table=1, n_packets=340, n_bytes=22460, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.2.0.2 actions=dec_ttl,mod_dl_src:00:00:00:00:02:05,resubmit(,2) cookie=0x0, duration=69.601s, table=1, n_packets=340, n_bytes=22460, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.2.0.2 actions=mod_dl_dst:00:00:00:00:00:02,resubmit(,2) cookie=0x0, duration=69.843s, table=1, n_packets=0, n_bytes=0, priority=10,arp,in_port=pf0vf2,arp_tpa=10.2.0.1,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:00:00:00:00:02:02,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x202->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xa020001->NXM_OF_ARP_SPA[],resubmit(,2) cookie=0x0, duration=69.746s, table=1, n_packets=1, n_bytes=60, priority=10,arp,in_port="br-int-patch",arp_tpa=10.0.0.3,arp_op=1 actions=move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[],mod_dl_src:00:00:00:00:04:01,load:0x2->NXM_OF_ARP_OP[],move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[],load:0x401->NXM_NX_ARP_SHA[],move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[],load:0xa000003->NXM_OF_ARP_SPA[],resubmit(,2) cookie=0x0, duration=69.684s, table=2, n_packets=830, n_bytes=1234642, priority=20,ip,metadata=0x1,in_port=pf0vf2,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2 actions=clone(ct_clear,load:0x2->OXM_OF_METADATA[],resubmit(,0)) cookie=0x0, duration=69.662s, table=2, n_packets=830, n_bytes=1234642, priority=30,ip,metadata=0x2,in_port=pf0vf2,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2 actions=ct(commit,table=3,nat(src=10.0.0.3)) cookie=0x0, duration=69.624s, table=2, n_packets=340, n_bytes=22460, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:04:01,nw_dst=10.2.0.2 actions=mod_dl_dst:00:00:00:00:02:04,resubmit(,3) cookie=0x0, duration=69.594s, table=2, n_packets=340, n_bytes=22460, priority=30,ip,metadata=0x1,in_port="br-int-patch",dl_dst=00:00:00:00:00:02,nw_dst=10.2.0.2 actions=clone(ct_clear,output:pf0vf2) cookie=0x0, duration=69.812s, table=2, n_packets=0, n_bytes=0, priority=10,arp,arp_spa=10.2.0.1 actions=IN_PORT cookie=0x0, duration=69.715s, table=2, n_packets=1, n_bytes=60, priority=10,arp,arp_spa=10.0.0.3 actions=IN_PORT cookie=0x0, duration=69.654s, table=3, n_packets=830, n_bytes=1234642, priority=30,ip,metadata=0x2,in_port=pf0vf2,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2 actions=clone(ct_clear,output:"br-int-patch") cookie=0x0, duration=69.616s, table=3, n_packets=340, n_bytes=22460, priority=20,ip,metadata=0x2,in_port="br-int-patch",dl_dst=00:00:00:00:02:04,nw_dst=10.2.0.2 actions=clone(ct_clear,load:0x1->OXM_OF_METADATA[],resubmit(,0)) ### debug using this command [root@bogon ~]# ovs-appctl ofproto/trace br-int in_port=pf0vf2,dl_src=00:00:00:00:00:02,dl_dst=00:00:00:00:02:02,tcp,nw_src=10.2.0.2,nw_dst=10.0.0.2,nw_ttl=64 Flow: tcp,in_port=3,vlan_tci=0x0000,dl_src=00:00:00:00:00:02,dl_dst=00:00:00:00:02:02,nw_src=10.2.0.2,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=64,tp_src=0,tp_dst=0,tcp_flags=0 bridge("br-int") ---------------- 0. ip,in_port=3,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2, priority 10 clone(load:0x1->OXM_OF_METADATA[],resubmit(,0)) load:0x1->OXM_OF_METADATA[] resubmit(,0) 0. ip,metadata=0x1,in_port=3,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2, priority 20 dec_ttl mod_dl_src:00:00:00:00:02:04 resubmit(,1) 1. ip,metadata=0x1,in_port=3,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2, priority 20 mod_dl_dst:00:00:00:00:02:05 resubmit(,2) 2. ip,metadata=0x1,in_port=3,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 20 clone(ct_clear,load:0x2->OXM_OF_METADATA[],resubmit(,0)) ct_clear load:0x2->OXM_OF_METADATA[] resubmit(,0) 0. ip,metadata=0x2,in_port=3,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 30 dec_ttl mod_dl_src:00:00:00:00:04:01 resubmit(,1) 1. ip,metadata=0x2,in_port=3,dl_dst=00:00:00:00:02:05,nw_dst=10.0.0.2, priority 30 mod_dl_dst:00:00:00:00:03:01 resubmit(,2) 2. ip,metadata=0x2,in_port=3,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2, priority 30 ct(commit,table=3,nat(src=10.0.0.3)) nat(src=10.0.0.3) -> A clone of the packet is forked to recirculate. The forked pipeline will be resumed at table 3. -> Sets the packet to an untracked state, and clears all the conntrack fields. Final flow: unchanged Megaflow: recirc_id=0,eth,ip,in_port=3,dl_src=00:00:00:00:00:02,dl_dst=00:00:00:00:02:02,nw_dst=10.0.0.2,nw_ttl=64,nw_frag=no Datapath actions: set(eth(src=00:00:00:00:04:01,dst=00:00:00:00:03:01)),set(ipv4(ttl=62)),ct(commit,nat(src=10.0.0.3)),recirc(0x32) =============================================================================== recirc(0x32) - resume conntrack with default ct_state=trk|new (use --ct-next to customize) Replacing src/dst IP/ports to simulate NAT: Initial flow: nw_src=10.2.0.2,tp_src=0,nw_dst=10.0.0.2,tp_dst=0 Modified flow: nw_src=10.0.0.3,tp_src=0,nw_dst=10.0.0.2,tp_dst=0 =============================================================================== Flow: recirc_id=0x32,ct_state=new|trk,eth,tcp,metadata=0x2,in_port=3,vlan_tci=0x0000,dl_src=00:00:00:00:04:01,dl_dst=00:00:00:00:03:01,nw_src=10.0.0.3,nw_dst=10.0.0.2,nw_tos=0,nw_ecn=0,nw_ttl=62,tp_src=0,tp_dst=0,tcp_flags=0 bridge("br-int") ---------------- thaw Resuming from table 3 3. ip,metadata=0x2,in_port=3,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2, priority 30 clone(ct_clear,output:5) ct_clear output:5 bridge("brp0") -------------- 0. in_port=3, priority 32768 output:1 Final flow: unchanged Megaflow: recirc_id=0x32,eth,ip,in_port=3,dl_dst=00:00:00:00:03:01,nw_dst=10.0.0.2,nw_frag=no Datapath actions: ct_clear,7 ### I cann't see difference between this and above ``` 4. My question: - Why this BUG happen? - Is there patch fix this BUG? Thanks~ ---- Simon Jones _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev