>
>Sorry for the 2nd email within 10 minutes – I realized, that I forgot to past
>ovs output for
>the dpdk0 and vhost-user interfaces:
>
>root@caesar:/home/cisco# ovs-ofctl show ovsdpdkbr0
>OFPT_FEATURES_REPLY (xid=0x2): dpid:000090e2ba99978c
>n_tables:254, n_buffers:256
>capabilities: FLOW_STATS TABLE_STATS PORT_STATS QUEUE_STATS ARP_MATCH_IP
>actions: output enqueue set_vlan_vid set_vlan_pcp strip_vlan mod_dl_src
>mod_dl_dst mod_nw_src
>mod_nw_dst mod_nw_tos mod_tp_src mod_tp_dst
> 1(dpdk0): addr:90:e2:ba:99:97:8c
> config: 0
> state: 0
> current: 10GB-FD
> speed: 10000 Mbps now, 0 Mbps max
> 2(vhost-user-1): addr:00:00:00:00:00:00
> config: PORT_DOWN
> state: LINK_DOWN
> speed: 0 Mbps now, 0 Mbps max
> LOCAL(ovsdpdkbr0): addr:90:e2:ba:99:97:8c
> config: PORT_DOWN
> state: LINK_DOWN
> current: 10MB-FD COPPER
> speed: 10 Mbps now, 0 Mbps max
>OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0
>root@caesar:/home/cisco#
>
>
>
>It looks, like vhost-user-1 is down and has a problem:
>
>root@caesar:/home/cisco# ovs-vsctl list interface vhost-user-1
>_uuid : d9922622-a6aa-4df3-b992-3658a57f71c3
>admin_state : down
>bfd : {}
>bfd_status : {}
>cfm_fault : []
>cfm_fault_status : []
>cfm_flap_count : []
>cfm_health : []
>cfm_mpid : []
>cfm_remote_mpids : []
>cfm_remote_opstate : []
>duplex : []
>error : []
>external_ids : {}
>ifindex : 0
>ingress_policing_burst: 0
>ingress_policing_rate: 0
>lacp_current : []
>link_resets : 3
>link_speed : []
>link_state : down
>lldp : {}
>mac : []
>mac_in_use : "00:00:00:00:00:00"
>mtu : 1500
>name : "vhost-user-1"
>ofport : 2
>ofport_request : []
>options : {}
>other_config : {}
>statistics : {rx_bytes=0, rx_errors=0, rx_packets=0, tx_bytes=0,
>tx_dropped=4012,
>tx_packets=0}
>status : {}
>type : dpdkvhostuser
>root@caesar:/home/cisco#
>
>
>dpdk0 Interfaces looks good:
>
>root@caesar:/home/cisco# ovs-vsctl list interface dpdk0
>_uuid : 4177750b-6bee-4daa-80ea-99f24f8dd38d
>admin_state : up
>bfd : {}
>bfd_status : {}
>cfm_fault : []
>cfm_fault_status : []
>cfm_flap_count : []
>cfm_health : []
>cfm_mpid : []
>cfm_remote_mpids : []
>cfm_remote_opstate : []
>duplex : full
>error : []
>external_ids : {}
>ifindex : 0
>ingress_policing_burst: 0
>ingress_policing_rate: 0
>lacp_current : []
>link_resets : 3
>link_speed : 10000000000
>link_state : up
>lldp : {}
>mac : []
>mac_in_use : "90:e2:ba:99:97:8c"
>mtu : 1500
>name : "dpdk0"
>ofport : 1
>ofport_request : []
>options : {}
>other_config : {}
>statistics : {rx_bytes=575477, rx_dropped=0, rx_errors=0,
>rx_packets=6785,
>tx_bytes=0, tx_dropped=0, tx_errors=0, tx_packets=0}
>status : {driver_name=rte_ixgbe_pmd, max_hash_mac_addrs="4096",
>max_mac_addrs="127", max_rx_pktlen="15872", max_rx_queues="128",
>max_tx_queues="128",
>max_vfs="0", max_vmdq_pools="64", min_rx_bufsize="1024", numa_id="1",
>pci-device_id="0x10fb",
>pci-vendor_id="0x32902", port_no="0"}
>type : dpdk
>root@caesar:/home/cisco#
>
>
>How can I bring vhost-user-1 up and get it sending packets to dpdk?
>
>
>Thanks again,
>Nikolai
>
>On 27.05.16, 22:50, "Nikolai Pitaev (npitaev)" <[email protected]> wrote:
>
>>Thanks, Mark!
>>You are absolutely right – lstopo showed the Intel NIC on NUMA node 1.
>>I was able to start OVS-DPDK with the following option in
>>/etc/default/openvswitch-switch
>>DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 0,2048 --vhost-owner libvirt-qemu:kvm
>>--vhost-perm 0664'
>>
>>Now I have a DIFFERENT problem – no connectivity from VM to the Outside World
>>via DPDK-OVS
>bridge.
>>
>>DPDK NIC Bindings
>>root@caesar:/home/cisco# dpdk_nic_bind --status
>>Network devices using DPDK-compatible driver
>>============================================
>>0000:8f:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=vfio-pci
>>unused=ixgbe
><-- Intel X520-DA2 10 GE Port 0
>>0000:8f:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=vfio-pci
>>unused=ixgbe <-
>- Intel X520-DA2 10GE Port 1
>>
>>Network devices using kernel driver
>>===================================
>>0000:0f:00.0 'I350 Gigabit Network Connection' if=enp15s0f0 drv=igb
>>unused=vfio-pci *Active*
>>…
>>root@caesar:/home/cisco#
>>
>>
>>I have two OVS-DPDK Bridges created as described in
>https://help.ubuntu.com/16.04/serverguide/DPDK.html
>>root@caesar:/home/cisco# ovs-vsctl show
>>cf57d236-c8ec-4099-a621-8fda17920828
>> Bridge "ovsdpdkbr0"
>> Port "dpdk0"
>> Interface "dpdk0" <-- this should be my
>> Intel X520 10GE
>Port 0
>> type: dpdk
>> Port "ovsdpdkbr0"
>> Interface "ovsdpdkbr0"
>> type: internal
>> Port "vhost-user-1" <-- this should be my
>> VM vNIC0
>> Interface "vhost-user-1"
>> type: dpdkvhostuser
>> Bridge "ovsdpdkbr1"
>> Port "ovsdpdkbr1"
>> Interface "ovsdpdkbr1"
>> type: internal
>> Port "vhost-user-2" <-- this should be my VM
>> vNIC1
>> Interface "vhost-user-2"
>> type: dpdkvhostuser
>> Port "dpdk1" <-- I hope, this is my
>> Intel X520 10GE
>Port 1
>> Interface "dpdk1"
>> type: dpdk
>> ovs_version: "2.5.0"
>>root@caesar:/home/cisco#
>>
>>
>>In my VM I have the following XML config for the vNICs:
>> <interface type='vhostuser'>
>> <mac address='52:54:00:2e:4e:e0'/>
>> <source type='unix' path='/var/run/openvswitch/vhost-user-1'
>> mode='client'/>
>> <model type='virtio'/>
>> <address type='pci' domain='0x0000' bus='0x00' slot='0x08'
>> function='0x0'/>
>> </interface>
>> <interface type='vhostuser'>
>> <mac address='52:54:00:95:c5:4f'/>
>> <source type='unix' path='/var/run/openvswitch/vhost-user-2'
>> mode='client'/>
>> <model type='virtio'/>
>> <address type='pci' domain='0x0000' bus='0x00' slot='0x09'
>> function='0x0'/>
>> </interface>
>>
>>Verified, that both sockets exist:
>>root@caesar:/home/cisco# ls -la /var/run/openvswitch/vhost-user-*
>>srw-rw-r-- 1 libvirt-qemu kvm 0 May 27 15:45 /var/run/openvswitch/vhost-user-1
>>srw-rw-r-- 1 libvirt-qemu kvm 0 May 27 15:49 /var/run/openvswitch/vhost-user-2
>>root@caesar:/home/cisco#
>>
>>VM successfully starts, I can bring both Interfaces up, BUT I can NOT ping
>>from my VM the
>outside world.
>>VM can ping fine using ANOTHER interface:
>> <interface type='network'>
>> <mac address='52:54:00:34:86:9c'/>
>> <source network='default'/>
>> <model type='virtio'/>
>> <address type='pci' domain='0x0000' bus='0x00' slot='0x03'
>> function='0x0'/>
>> </interface>
>>
>>So, the problem is the connectivity via OVS-DPDK. Simple ping works.
Have you tried adding explicit flows to direct traffic to the appropriate
interfaces, as opposed to using the NORMAL action?
This isn't a proper solution as such, but may enable traffic to pass
nonetheless.
>>
>>
>>What is the best way to troubleshoot connectivity issues with OVS-DPDK?
>>With native OVS I simple did “ifconfig vnet1” and immediately saw packets and
>>drops. Now the
>interfaces like dpdk0 and vhost-user-1 are NOT visible for ifconfig.
Have you tried the following?
- ovs-vsctl get Interface vhost-user-1 stats # view interface stats
- ovs-appctl dpctl/dump-flows # dump 'flattened' datapath
flows
=> traffic must be live when running this command, as it dumps the datapath
flows; examining the underlying dp flows, you may spot that some flows are not
as you expect (i.e. traffic may be dropped, etc.)
>>
>>root@caesar:/home/cisco# ovs-dpctl show
>>system@ovs-system:
>> lookups: hit:0 missed:0 lost:0
>> flows: 0
>> masks: hit:0 total:1 hit/pkt:0.00
>> port 0: ovs-system (internal)
>>root@caesar:/home/cisco#
>>
>>
>>Thanks,
>>Nikolai
>>
>>
>>On 27.05.16, 11:09, "Kavanagh, Mark B" <[email protected]> wrote:
>>
>>>>
>>>>Hi!
>>>>
>>>>I try to install and use OVS with DPDK on Ubuntu 16.04 following this guide:
>>>>https://help.ubuntu.com/16.04/serverguide/DPDK.html
>>>>
>>>>On a Cisco UCS C240 with two physical CPUs (18 Cores each) I have two Intel
>>>>X520-DA2
>>>>Cards, which is recognized and show properly:
>>>>root@caesar:/home/cisco# dpdk_nic_bind --status
>>>>Network devices using DPDK-compatible driver
>>>>============================================
>>>>0000:8f:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=vfio-pci
>>>>unused=ixgbe <- looks good, vfio-pci driver shown properly
>>>>0000:8f:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=vfio-pci
>>>>unused=ixgbe <- looks good, vfio-pci driver shown properly
>>>>Network devices using kernel driver
>>>>===================================
>>>>0000:07:00.0 'VIC Ethernet NIC' if=enp7s0 drv=enic unused=vfio-pci
>>>>0000:08:00.0 'VIC Ethernet NIC' if=enp8s0 drv=enic unused=vfio-pci
>>>>0000:0f:00.0 'I350 Gigabit Network Connection' if=enp15s0f0 drv=igb
>>>>unused=vfio-pci
>>>>*Active*
>>>>…
>>>>Other network devices
>>>>=====================
>>>><none>
>>>>root@caesar:/home/cisco#
>>>>
>>>>If I tweak the OVS Config as described in the Ubuntu DPDK Gude with the
>>>>following line
>>>> echo "DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 2048 --vhost-owner
>>>> libvirt-qemu:kvm --vhost-perm
>>>>0664'" | sudo tee -a /etc/default/openvswitch-switch
>>>>I will get the following error message:
>>>>root@caesar:/home/cisco# ovs-vsctl show
>>>>cf57d236-c8ec-4099-a621-8fda17920828
>>>> Bridge "ovsdpdkbr0"
>>>> Port "dpdk0"
>>>> Interface "dpdk0"
>>>> type: dpdk
>>>> error: "could not open network device dpdk0 (Cannot
>>>> allocate memory)"
>>>> Port "ovsdpdkbr0"
>>>> Interface "ovsdpdkbr0"
>>>> type: internal
>>>> ovs_version: "2.5.0"
>>>>root@caesar:/home/cisco#
>>>>
>>>>My UCS C240 Server has two nodes with 18 cores each. In the following forum
>>>>http://comments.gmane.org/gmane.linux.network.openvswitch.general/6760
>>>>I saw similar issue and the solution was to configure memory like this:
>>>>---
>>>>Start vswitchd process with 8GB on each numa node (if reserve memory on
>>>>just 1 numa node,
>>>>creating dpdk port will fail: cannot allocate memory)
>>>>./vswitchd/ovs-vswitchd --dpdk -c 0x1 -n 4 --socket-mem 8192,8192 --
>>>>unix:/usr/local/var/run/openvswitch/db.sock --pidfile --detach
>>>>---
>>>>
>>>>If I change /etc/default/openvswitch-switch to
>>>> DPDK_OPTS='--dpdk -c 0x1 -n 4 --socket-mem 4096,4096 --vhost-owner
>>>> libvirt-qemu:kvm --
>>>>vhost-perm 0664'
>>>>then I can enter OVS CLI commandos, but have to use “Ctrl +C” to get prompt
>>>>after any OVS
>>>>CLI. But it looks, like OVS accepts and executes CLIs.
>>>>I can create OVS DPDK bridges, but OVS cannot create a vhost_user socket at
>>>>/var/run/openvswitch/vhost-user-1 – the following CLI does not work:
>>>>
>>>>cisco@caesar:~$ sudo ovs-vsctl add-port ovsdpdkbr1 vhost-user-1 -- set
>>>>Interface vhost-
>>>>user-1 type=dpdkvhostuser
>>>>^C2016-05-26T17:11:16Z|00002|fatal_signal|WARN|terminating with signal 2
>>>>(Interrupt)
>>>>
>>>>cisco@caesar:~$ sudo ovs-vsctl show
>>>>cf57d236-c8ec-4099-a621-8fda17920828
>>>> Bridge "ovsdpdkbr2"
>>>> Port "ovsdpdkbr2"
>>>> Interface "ovsdpdkbr2"
>>>> type: internal
>>>> Port "dpdk1"
>>>> Interface "dpdk1"
>>>> type: dpdk
>>>> Bridge "ovsdpdkbr1"
>>>> Port "vhost-user-1"
>>>> Interface "vhost-user-1"
>>>> type: dpdkvhostuser
>>>> Port "ovsdpdkbr1"
>>>> Interface "ovsdpdkbr1"
>>>> type: internal
>>>> Port "dpdk0"
>>>> Interface "dpdk0"
>>>> type: dpdk
>>>> ovs_version: "2.5.0"
>>>>cisco@caesar:~$
>>>>
>>>>There is NO vhost-user-1 in /var/run/openvswitch/
>>>>cisco@caesar:~$ ls -la /var/run/openvswitch/
>>>>total 4
>>>>drwxr-xr-x 2 root root 100 May 26 11:51 .
>>>>drwxr-xr-x 27 root root 1040 May 26 12:06 ..
>>>>srwxr-x--- 1 root root 0 May 26 11:49 db.sock
>>>>srwxr-x--- 1 root root 0 May 26 11:49 ovsdb-server.5559.ctl
>>>>-rw-r--r-- 1 root root 5 May 26 11:49 ovsdb-server.pid
>>>>cisco@caesar:~$
>>>>cisco@caesar:~$
>>>>
>>>>
>>>>So, my questions are:
>>>>1. What is the right config line for servers with two physical CPU (in my
>>>>case node0 and
>>>>node1 with 18 CPUs each) for
>>>>echo "DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 2048 --vhost-owner libvirt-qemu:kvm
>>>>--vhost-perm
>>>>0664'" | sudo tee -a /etc/default/openvswitch-switch
>>>
>>>Hi Nikolai,
>>>
>>>You mentioned that when you specify the -m argument as '2048', you cannot
>>>add dpdk0, but
>when you specify "-m 4096, 4096" (i.e. 4k for NUMA node 0, 4k for NUMA node
>1)that dpdk phy
>ports are added successfully.
>>>This leads me to believe that your NICs are installed on the PCI slots for
>>>NUMA node 1 -
>this is easily confirmed by use of the 'lstopo' tool, part of the 'hwloc'
>package:
>https://www.open-mpi.org/projects/hwloc/.
>>>To correct this, either move your NICs to the PCI slots for NUMA node 0, or
>>>change your -m
>argument to "0, 2048".
>>>
>>>Hope this helps,
>>>Mark
>>>
>>>>
>>>>2. How can OVS create a vhost_user socket at
>>>>/var/run/openvswitch/vhost-user-1 ?
>>>>
>>>>
>>>>
>>>>And yes, HugePage support is enabled:
>>>>root@caesar:/home/cisco# cat /proc/meminfo | grep Huge
>>>>AnonHugePages: 16384 kB
>>>>HugePages_Total: 64
>>>>HugePages_Free: 0
>>>>HugePages_Rsvd: 0
>>>>HugePages_Surp: 0
>>>>Hugepagesize: 2048 kB
>>>>root@caesar:/home/cisco#
>>>>
>>>>In /etc/default/grub I have:
>>>>GRUB_CMDLINE_LINUX_DEFAULT="iommu=pt intel_iommu=on hugepages=8192
>>>>hugepagesz=1G
>>>>hugepages=8 isolcpus=4,5,6,7,8"
>>>>
>>>>
>>>>Thanks,
>>>>Nikolai
>>
_______________________________________________
discuss mailing list
[email protected]
http://openvswitch.org/mailman/listinfo/discuss
