Hi satish,
I think the problem is in the rules of the form
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:50,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:195, bytes:66690, used:3.722s, actions:1,3,4
Any idea why all ipv4 rules has broadcast mac address? and flooded to all ports?
Anyway, flooding is not supported currently it requires that we will clone the
packet, and
It will be supported on next release (high probability), but ctx5 and above as
far as I know.
BR,
Roni
From: satish dhote <sdhote...@gmail.com>
Sent: Wednesday, February 26, 2020 3:27 PM
To: Roni Bar Yanai <ron...@mellanox.com>
Cc: Moshe Levi <mosh...@mellanox.com>; ovs-discuss@openvswitch.org
Subject: Re: [ovs-discuss] TC flow offload using OVS - failed to offload flow:
Invalid argument: enp0s25
Answer inline and provided other config related information below. Let me know
in case you need any other information.
eth1 is it the port representor of VF1?
eth0 and eth1 are port representor for VF1 and VF2 respectively.
eth0 is the up-link representor?
ens11 is pf(uplink) interface.
Can you share dpif/dump-flows so we can see what are the flows you are trying
to offload?
[root@localhost ~]# ovs-appctl dpif/dump-flows ovs-sriov
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:63175587, bytes:21606050754, used:0.000s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:50,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:8597565, bytes:773780850, used:0.000s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=33:33:ff:db:1e:09),eth_type(0x86dd),ipv6(frag=no),
packets:209474316, bytes:18014791176, used:0.000s, actions:1,3,4
recirc_id(0),in_port(3),eth(src=e4:11:22:33:44:50,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:3, bytes:1026, used:5.357s, actions:1,2,4
recirc_id(0),in_port(2),eth(src=9a:d7:9b:84:3a:5e,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:355100402, bytes:31959036180, used:0.000s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:50,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:0, bytes:0, used:never, actions:1,3,4
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:152331284, bytes:13709822800, used:0.001s, actions:1,3,4
[root@localhost ~]# ovs-ofctl dump-flows ovs-sriov
cookie=0x0, duration=974.646s, table=0, n_packets=0, n_bytes=0, priority=0
actions=NORMAL
[root@localhost ~]# ovs-appctl dpif/dump-flows type=offloaded ovs-sriov
(Note sure whether this type parameter works here for hardware offloaded
flows:) )
recirc_id(0),in_port(4),eth(src=e4:11:22:33:44:51,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:3, bytes:1026, used:7.092s, actions:1,2,3
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:209960759, bytes:71806579578, used:0.000s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:50,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:151469827, bytes:13632285570, used:0.000s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=33:33:ff:db:1e:09),eth_type(0x86dd),ipv6(frag=no),
packets:60983904, bytes:5244615744, used:0.001s, actions:1,3,4
recirc_id(0),in_port(3),eth(src=e4:11:22:33:44:50,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:4, bytes:1368, used:3.748s, actions:1,2,4
recirc_id(0),in_port(2),eth(src=a2:b9:ce:03:0f:ba,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:18142152, bytes:1632793680, used:0.001s, actions:1,3,4
recirc_id(0),in_port(4),eth(src=e4:11:22:33:44:51,dst=33:33:00:00:00:02),eth_type(0x86dd),ipv6(frag=no),
packets:2, bytes:124, used:4.037s, actions:1,2,3
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:51,dst=33:33:00:00:00:02),eth_type(0x86dd),ipv6(frag=no),
packets:890, bytes:55180, used:7.732s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=9a:d7:9b:84:3a:5e,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:497909550, bytes:44811859500, used:0.001s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=e4:11:22:33:44:50,dst=ff:ff:ff:ff:ff:ff),eth_type(0x0800),ipv4(frag=no),
packets:195, bytes:66690, used:3.722s, actions:1,3,4
recirc_id(0),in_port(2),eth(src=50:6b:4b:0c:81:70,dst=33:33:00:00:00:16),eth_type(0x86dd),ipv6(frag=no),
packets:309799865, bytes:27881997410, used:0.000s, actions:1,3,4
TC commands shows empty response.
[root@localhost ~]# tc filter show dev ovs-sriov ingress
[root@localhost ~]# tc filter show dev eth0 ingress
[root@localhost ~]# tc filter show dev eth1 ingress
[root@localhost ~]# tc filter show dev ens11 ingress
[root@localhost ~]# ovs-dpctl show
2020-02-26T13:10:28Z|00001|dpif_netlink|INFO|The kernel module does not support
meters.
system@ovs-system:
lookups: hit:2826089018 missed:439 lost:0
flows: 8
masks: hit:5034569510 total:2 hit/pkt:1.78
port 0: ovs-system (internal)
port 1: ovs-sriov (internal)
port 2: ens11
port 3: eth0
port 4: eth1
Is traffic running? Ping replay and ssh?
Yes. below are the response.
VM1 ip: 192.168.122.115
VM2 ip: 192.168.122.116
[root@localhost ~]# ping 192.168.122.116
PING 192.168.122.116 (192.168.122.116) 56(84) bytes of data.
64 bytes from
192.168.122.116<https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2F192.168.122.116%2F&data=02%7C01%7Croniba%40mellanox.com%7Cb4addd6f0606436fa3c608d7babf9f72%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C1%7C637183204488838419&sdata=WnrK7QlqOM7YrMF80vkPo6Kvjmu8gSXKJaSuU6Numvk%3D&reserved=0>:
icmp_seq=1 ttl=64 time=3.08 ms
64 bytes from
192.168.122.116<https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2F192.168.122.116%2F&data=02%7C01%7Croniba%40mellanox.com%7Cb4addd6f0606436fa3c608d7babf9f72%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C1%7C637183204488838419&sdata=WnrK7QlqOM7YrMF80vkPo6Kvjmu8gSXKJaSuU6Numvk%3D&reserved=0>:
icmp_seq=2 ttl=64 time=2.54 ms
[root@localhost ~]# ssh root@192.168.122.116<mailto:root@192.168.122.116>
The authenticity of host '192.168.122.116 (192.168.122.116)' can't be
established.
ECDSA key fingerprint is 0b:e0:20:e6:97:16:35:86:ca:3c:1b:86:5a:36:9e:3a.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.122.116' (ECDSA) to the list of known hosts.
root@192.168.122.116<mailto:root@192.168.122.116>'s password:
Error logs from ovs-vswitchd.log:
2020-02-26T13:13:02.750Z|00038|dpif_netlink(handler11)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:13:02.786Z|00100|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:13:02.789Z|00101|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: ens11
2020-02-26T13:13:03.638Z|00102|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:13:04.643Z|00078|dpif_netlink(handler9)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:13:08.645Z|00055|dpif_netlink(handler10)|ERR|failed to offload
flow: Invalid argument: ens11
2020-02-26T13:13:11.769Z|00103|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth1
2020-02-26T13:13:22.691Z|00079|dpif_netlink(handler9)|ERR|Dropped 10 log
messages in last 11 seconds (most recently, 8 seconds ago) due to excessive rate
2020-02-26T13:13:22.691Z|00080|dpif_netlink(handler9)|ERR|failed to offload
flow: Invalid argument: ens11
2020-02-26T13:13:29.479Z|00104|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:13:43.924Z|00105|dpif_netlink(handler8)|ERR|Dropped 3 log
messages in last 15 seconds (most recently, 11 seconds ago) due to excessive
rate
2020-02-26T13:13:43.924Z|00106|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth1
2020-02-26T13:13:56.780Z|00056|dpif_netlink(handler10)|ERR|Dropped 7 log
messages in last 13 seconds (most recently, 7 seconds ago) due to excessive rate
2020-02-26T13:13:56.781Z|00057|dpif_netlink(handler10)|ERR|failed to offload
flow: Invalid argument: eth1
2020-02-26T13:14:15.495Z|00107|dpif_netlink(handler8)|ERR|Dropped 6 log
messages in last 18 seconds (most recently, 12 seconds ago) due to excessive
rate
2020-02-26T13:14:15.495Z|00108|dpif_netlink(handler8)|ERR|failed to offload
flow: Invalid argument: eth0
2020-02-26T13:14:29.165Z|00081|dpif_netlink(handler9)|ERR|failed to offload
flow: Invalid argument: eth1
2020-02-26T13:14:29.169Z|00082|dpif_netlink(handler9)|ERR|failed to offload
flow: Invalid argument: ens11
2020-02-26T13:14:41.783Z|00058|dpif_netlink(handler10)|ERR|Dropped 6 log
messages in last 9 seconds (most recently, 4 seconds ago) due to excessive rate
[root@localhost ~]# ovs-vsctl get Open_vSwitch . other_config:hw-offload
"true"
[root@localhost ~]# ethtool -k ens11 | grep hw-tc-offload
hw-tc-offload: on [requested off] (Not sure what
is "requested off" here)
[root@localhost ~]# ethtool -k eth0 | grep hw-tc-offload
hw-tc-offload: on
[root@localhost ~]# ethtool -k eth1 | grep hw-tc-offload
hw-tc-offload: on
Thanks
Satish
On Wed, Feb 26, 2020 at 1:10 PM Roni Bar Yanai
<ron...@mellanox.com<mailto:ron...@mellanox.com>> wrote:
Hi,
eth1 is it the port representor of VF1?
eth0 is the up-link representor?
Can you share dpif/dump-flows so we can see what are the flows
you are trying to offload? Not all flows are supported.
Is traffic running? Ping replay and ssh?
Thanks.
From: satish dhote <sdhote...@gmail.com<mailto:sdhote...@gmail.com>>
Sent: Wednesday, February 26, 2020 8:39 AM
To: Moshe Levi <mosh...@mellanox.com<mailto:mosh...@mellanox.com>>
Cc: Roni Bar Yanai <ron...@mellanox.com<mailto:ron...@mellanox.com>>;
ovs-discuss@openvswitch.org<mailto:ovs-discuss@openvswitch.org>
Subject: Re: [ovs-discuss] TC flow offload using OVS - failed to offload flow:
Invalid argument: enp0s25
Hi Moshe,
Sorry for the confusion. My recent response has new setup details, so please
ignore the instructions and setup details which I sent as part of very first
email.
Here is the fresh setup details and logs, and yes, ens11 is a physical link.
Openvswitch version - "2.12.0"
Host machine -
OS - Centos7.5
mellanox card - ConnectX-4 Lx
Physical interface ens11
Virtual machines -
2 VM's
OS - Minimal Centos7.5
Attached each VF to each virtual machine
Steps followed to enable sriov and configuring ovs offload
# lspci | grep -i mell
01:00.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx]
# ethtool -i ens11 | head -5
driver: mlx5_core
version: 4.7-3.2.9
firmware-version: 14.26.4012 (MT_2410110034)
expansion-rom-version:
bus-info: 0000:01:00.0
# cat /sys/class/net/ens11/device/sriov_totalvfs
4
# echo 2 > /sys/class/net/ens11/device/sriov_numvfs
# lspci | grep -i mell
01:00.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx]
01:00.1 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx Virtual Function]
01:00.2 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx Virtual Function]
#cat /sys/class/net/ens11/device/sriov_numvfs
2
# echo nic_netdev > /sys/class/net/ens11/compat/devlink/uplink_rep_mode
# echo 0000:01:00.1 > /sys/bus/pci/drivers/mlx5_core/unbind
# echo 0000:01:00.2 > /sys/bus/pci/drivers/mlx5_core/unbind
# echo switchdev > /sys/class/net/ens11/compat/devlink/mode
# systemctl start openvswitch
# ovs-vsctl add-br ovs-sriov
# ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
# systemctl restart openvswitch
# ovs-vsctl add-port ovs-sriov ens11
# ovs-vsctl add-port ovs-sriov eth0
# ovs-vsctl add-port ovs-sriov eth1
# ovs-vsctl show
8bcbcf4b-e1df-4e79-b218-075d9d77568b
Bridge ovs-sriov
Port "ens11"
Interface "ens11"
Port "eth1"
Interface "eth1"
Port ovs-sriov
Interface ovs-sriov
type: internal
Port "eth0"
Interface "eth0"
Traffic type from one from one VM to another VM:
ICMP(command: ping), TCP(command: ssh)
Logs:
dpif_netlink(handler6)|ERR|Dropped 5 log messages in last 7 seconds (most
recently, 3 seconds ago) due to excessive rate
dpif_netlink(handler6)|ERR|failed to offload flow: Invalid argument: ens11
dpif_netlink(handler6)|ERR|Dropped 3 log messages in last 8 seconds (most
recently, 1 seconds ago) due to excessive rate
dpif_netlink(handler6)|ERR|failed to offload flow: Invalid argument: eth1
Thanks
Satish
On Wed, Feb 26, 2020 at 11:38 AM Moshe Levi
<mosh...@mellanox.com<mailto:mosh...@mellanox.com>> wrote:
You uplink (or physical link) is ens11 so why did you add enp0s25 to the
bridge?
From: discuss
<ovs-discuss-boun...@openvswitch.org<mailto:ovs-discuss-boun...@openvswitch.org>>
On Behalf Of satish dhote
Sent: Wednesday, February 26, 2020 8:01 AM
To: Roni Bar Yanai <ron...@mellanox.com<mailto:ron...@mellanox.com>>
Cc: ovs-discuss@openvswitch.org<mailto:ovs-discuss@openvswitch.org>
Subject: Re: [ovs-discuss] TC flow offload using OVS - failed to offload flow:
Invalid argument: enp0s25
Hi Roni,
Thanks for your response.
Below are the setup details.
Openvswitch version - "2.12.0"
Host machine -
OS - Centos7.5
mellanox card - ConnectX-4 Lx
Physical interface ens11
Virtual machines -
2 VM's
OS - Minimal Centos7.5
Attached each VF to each virtual machine
Steps followed to enable sriov and configuring ovs offload
# lspci | grep -i mell
01:00.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx]
# ethtool -i ens11 | head -5
driver: mlx5_core
version: 4.7-3.2.9
firmware-version: 14.26.4012 (MT_2410110034)
expansion-rom-version:
bus-info: 0000:01:00.0
# cat /sys/class/net/ens11/device/sriov_totalvfs
4
# echo 2 > /sys/class/net/ens11/device/sriov_numvfs
# lspci | grep -i mell
01:00.0 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx]
01:00.1 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx Virtual Function]
01:00.2 Ethernet controller: Mellanox Technologies MT27710 Family [ConnectX-4
Lx Virtual Function]
#cat /sys/class/net/ens11/device/sriov_numvfs
2
# echo nic_netdev > /sys/class/net/ens11/compat/devlink/uplink_rep_mode
# echo 0000:01:00.1 > /sys/bus/pci/drivers/mlx5_core/unbind
# echo 0000:01:00.2 > /sys/bus/pci/drivers/mlx5_core/unbind
# echo switchdev > /sys/class/net/ens11/compat/devlink/mode
# systemctl start openvswitch
# ovs-vsctl add-br ovs-sriov
# ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
# systemctl restart openvswitch
# ovs-vsctl add-port ovs-sriov ens11
# ovs-vsctl add-port ovs-sriov eth0
# ovs-vsctl add-port ovs-sriov eth1
Traffic type from one from one VM to another VM: ICMP(command: ping),
TCP(command: ssh)
let me know if I'm missing anything.
Thanks
Satish
On Mon, Feb 24, 2020 at 2:50 PM Roni Bar Yanai
<ron...@mellanox.com<mailto:ron...@mellanox.com>> wrote:
Hi Satish,
Can you share more details of your setup. How it is connected and what is the
traffic you run?
Are you in switchdev mode?
See
https://www.mellanox.com/related-docs/prod_software/ASAP2_Hardware_Offloading_for_vSwitches_User_Manual_v4.4.pdf
BR,
Roni
Hi Team,
I'm using OVS to understand its feature and workflow and came across the topic
of "offloading flow in hardware" through TC.
I installed OVS following official guide
http://docs.openvswitch.org/en/latest/intro/install/general/#bootstrapping<https://eur03.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdocs.openvswitch.org%2Fen%2Flatest%2Fintro%2Finstall%2Fgeneral%2F%23bootstrapping&data=02%7C01%7Croniba%40mellanox.com%7Cb4addd6f0606436fa3c608d7babf9f72%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C1%7C637183204488848410&sdata=4CMhcIB9rvTbfGOw6XJTSzSHXrng%2BaWAI1YR5nbptXY%3D&reserved=0>
https://docs.mellanox.com/display/MLNXOFEDv473290/OVS+Offload+Using+ASAP2+Direct#OVSOffloadUsingASAP2Direct-Overview
I tried to add flows after enabling hw_offload and then added few basic flow
using ovs-ofctl utilities. Flows are getting added into ovs-datapath but not
getting offloaded into hardware.
I'm using mellanox card connectx4, centos 7.5, ovs 2.1.12.
Please let me know whether am I missing something? OR this could be a bug?
Below are the various command output for the references, let me know for
anything else.
ovs-vsctl set Open_vSwitch . other_config:hw-offload=true
root@satish:~# ovs-vsctl show
966880f9-7702-44b9-be38-3dfc2d3cfb44
Bridge test-br
Port "enp0s25"
Interface "enp0s25"
Port test-br
Interface test-br
type: internal
ovs_version: "2.12.0"
root@satish:~# ovs-ofctl dump-flows test-br
cookie=0x0, duration=26540.500s, table=0, n_packets=0, n_bytes=0, in_port=1
actions=output:enp0s25
cookie=0x0, duration=31399.514s, table=0, n_packets=18537, n_bytes=2420553,
priority=0 actions=NORMAL
root@satish:~# tc filter show dev test-br ingress
root@satish:~#
root@satish:~# tc filter show dev enp0s25 ingress
satish@satish:~$ sudo tail -f /usr/local/var/log/openvswitch/ovs-vswitchd.log
..................
2020-02-20T05:31:20.155Z|00255|dpif_netlink(handler6)|ERR|Dropped 5 log
messages in last 7 seconds (most recently, 3 seconds ago) due to excessive rate
2020-02-20T05:31:20.155Z|00256|dpif_netlink(handler6)|ERR|failed to offload
flow: Invalid argument: enp0s25
2020-02-20T05:31:30.516Z|00257|dpif_netlink(handler6)|ERR|Dropped 3 log
messages in last 8 seconds (most recently, 1 seconds ago) due to excessive rate
2020-02-20T05:31:30.516Z|00258|dpif_netlink(handler6)|ERR|failed to offload
flow: Invalid argument: enp0s25
Thanks
Satish
_______________________________________________
discuss mailing list
disc...@openvswitch.org<mailto:disc...@openvswitch.org>
https://mail.openvswitch.org/mailman/listinfo/ovs-discuss<https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fmail.openvswitch.org%2Fmailman%2Flistinfo%2Fovs-discuss&data=02%7C01%7Croniba%40mellanox.com%7Cb4addd6f0606436fa3c608d7babf9f72%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C1%7C637183204488858402&sdata=U7vCpuaynnqCvda3Ru6bb5iE6lAv4hxJgcsceAvJCjs%3D&reserved=0>
_______________________________________________
discuss mailing list
disc...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-discuss
