This patch refactors the DPDK offload test infrastructure to simplify the interface configuration. Instead of using OVS_DPDK_VF_PCI_ADDRS with a space-separated list of PCI addresses and VF indices (e.g., "0000:17:00.0,0 0000:17:00.0,1"), the tests now use OVS_PF_PCI with a single PF address (e.g., "0000:17:00.0") and derive VF information from sysfs.
Signed-off-by: Eelco Chaudron <[email protected]> --- Changes to v3: - Fixed documentation. - Changed test macro names from xxxx_netdev() to xxx_port(). - Use local variables in shell macros. Changes to v2: - Use PCI address rather than port name, so other blades might be supported. - Moved/renamed shell helpers. - Updated documentation. --- Documentation/topics/testing.rst | 24 +++--- tests/system-common-macros.at | 90 ++++++++++++++++++++++ tests/system-dpdk-offloads-macros.at | 108 ++++----------------------- 3 files changed, 120 insertions(+), 102 deletions(-) diff --git a/Documentation/topics/testing.rst b/Documentation/topics/testing.rst index deb6088d7..abe670b8b 100644 --- a/Documentation/topics/testing.rst +++ b/Documentation/topics/testing.rst @@ -306,19 +306,25 @@ Userspace datapath with DPDK offload To invoke the userspace datapath tests with DPDK and its rte_flow offload, the same prerequisites apply as above. In addition, six Virtual Function (VF) -interfaces must be preconfigured and capable of hardware offloading traffic -between each other. +interfaces must be preconfigured on a single Physical Function (PF) that +supports rte_flow hardware offload. -These six VFs need to be passed as a list of PF PCI addresses with their -corresponding VF indexes in the OVS_DPDK_VF_PCI_ADDRS variable. -For example:: +This is an example on how to set this up for an NVIDIA blade on port +``ens2f0np0``:: - OVS_DPDK_VF_PCI_ADDRS="0000:17:00.0,0 0000:17:00.0,1 0000:17:00.0,2 0000:17:00.0,3 0000:17:00.0,4 0000:17:00.0,5" + OVS_PF_PCI=$(basename $(readlink /sys/class/net/ens2f0np0/device)) + echo 0 > /sys/bus/pci/devices/$OVS_PF_PCI/sriov_numvfs + devlink dev eswitch set pci/$OVS_PF_PCI mode switchdev + echo 6 > /sys/bus/pci/devices/$OVS_PF_PCI/sriov_numvfs -To invoke the dpdk offloads testsuite with the userspace datapath, run:: +This PF's PCI ID needs to be passed with the OVS_PF_PCI variable. +To invoke the DPDK offloads testsuite with the userspace datapath, run:: - make check-dpdk-offloads \ - OVS_DPDK_VF_PCI_ADDRS="0000:17:00.0,0 0000:17:00.0,1 0000:17:00.0,2 0000:17:00.0,3 0000:17:00.0,4 0000:17:00.0,5" + make check-dpdk-offloads OVS_PF_PCI=0000:17:00.0 + +.. note:: + This has only been tested on NVIDIA blades due to the limited availability + of other blades that support rte_flow. Userspace datapath: Testing and Validation of CPU-specific Optimizations ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/tests/system-common-macros.at b/tests/system-common-macros.at index e4862231a..97b80405b 100644 --- a/tests/system-common-macros.at +++ b/tests/system-common-macros.at @@ -409,3 +409,93 @@ m4_define([OVS_CHECK_PSAMPLE], # OVS_CHECK_XT() m4_define([OVS_CHECK_XT], [AT_SKIP_IF([test $HAVE_IPTABLES = no && test $HAVE_NFT = no])]) + +OVS_START_SHELL_HELPERS + +# ovs_get_vf_port(<PF_PORT_PCI>, <VF_INDEX>) +# +# This function tries to find the VF's netdev for the given PF's VF index. +# +ovs_get_vf_port() { + local PF_PCI=$1 + local VF_IDX=$2 + + local vf_net_dir="/sys/bus/pci/devices/$PF_PCI/virtfn$VF_IDX/net" + if test ! -d "$vf_net_dir"; then + return 1 + fi + + local vf_name=$(ls "$vf_net_dir" 2>/dev/null | head -1) + if test -z "$vf_name"; then + return 2 + fi + + echo "$vf_name" + return 0 +} + +# ovs_get_representor_port(<PF_PORT_PCI>, <VF_INDEX>) +# +# This function tries to find the representor's netdev for the given PF's VF +# index. +# +ovs_get_representor_port() { + local PF_PCI=$1 + local VF_IDX=$2 + + local representor=$(grep -l "vf$VF_IDX$" \ + /sys/bus/pci/devices/$PF_PCI/net/*/phys_port_name \ + 2>/dev/null | head -1) + if test -z "$representor"; then + return 2 + fi + + basename $(dirname "$representor") + return 0 +} + +# ovs_verify_pf_cfg(<PF_PORT_PCI>) +# +# Verify that the given PF port exists and that the required VFs are properly +# configured, i.e. have representor ports. +# +ovs_verify_pf_cfg() { + local PF_PCI=$1 + local VF_NAMES=() + local REPRESENTOR_NAMES=() + local PCI_REGEX=[[0-9a-fA-F]]{4}:[[0-9a-fA-F]]{2}:[[0-9a-fA-F]]{2}\.[[0-7]] + local vf_net_dir vf_name representor i + + echo "$PF_PCI" | grep -E -q "^$PCI_REGEX$" || return 1 + + for i in {0..5}; do + # Get VF netdev using PCI path + vf_net_dir="/sys/bus/pci/devices/$PF_PCI/virtfn$i/net" + if test ! -d "$vf_net_dir"; then + return 2 + fi + + vf_name=$(ls "$vf_net_dir" 2>/dev/null | head -1) + if test -z "$vf_name"; then + return 3 + fi + VF_NAMES+=($vf_name) + + # Find representor port for this VF + representor=$(grep -l "vf$i$" \ + /sys/bus/pci/devices/$PF_PCI/net/*/phys_port_name \ + 2>/dev/null | head -1) + if test -z "$representor"; then + return 4 + fi + REPRESENTOR_NAMES+=($(basename $(dirname "$representor"))) + done + + if test "${#VF_NAMES[*]}" -lt 6 -o "${#REPRESENTOR_NAMES[*]}" -lt 6; then + return 5 + fi + + return 0 +} + +OVS_END_SHELL_HELPERS diff --git a/tests/system-dpdk-offloads-macros.at b/tests/system-dpdk-offloads-macros.at index ff7a6e095..5aa19112a 100644 --- a/tests/system-dpdk-offloads-macros.at +++ b/tests/system-dpdk-offloads-macros.at @@ -54,10 +54,8 @@ m4_define([CHECK_NO_DPDK_OFFLOAD], # m4_define([OVS_DPDK_OFFLOAD_PRE_CHECK], [ OVS_DPDK_PRE_CHECK() - AT_SKIP_IF( - [test "$(printf '%s' "$OVS_DPDK_VF_PCI_ADDRS" | wc -w)" -ne 6]) - - AT_SKIP_IF([! ovs_dpdk_verify_vf_cfg "$OVS_DPDK_VF_PCI_ADDRS"]) + AT_SKIP_IF([test -z "$OVS_PF_PCI"]) + AT_SKIP_IF([! ovs_verify_pf_cfg "$OVS_PF_PCI"]) ]) # OVS_TRAFFIC_VSWITCHD_START([vsctl-args], [vsctl-output], [dbinit-aux-args]) @@ -114,39 +112,38 @@ $1";/mlx5_net: Failed to update link status: /d"]) m4_define([ADD_VF], [ USER_PORT=$1 case "$USER_PORT" in - client) PORT_NO=0 ;; - server) PORT_NO=1 ;; - *) PORT_NO=${USER_PORT##*[!0-9]} ;; + client) VF_IDX=0 ;; + server) VF_IDX=1 ;; + *) VF_IDX=${USER_PORT##*[!0-9]} ;; esac - AT_CHECK([[[ "$PORT_NO" -ge 0 ]] && [[ "$PORT_NO" -le 6 ]] || return 66]) - PORT_CFG=$(echo $OVS_DPDK_VF_PCI_ADDRS | cut -d' ' -f$((PORT_NO+1))) - PF_PCI=$(ovs_dpdk_get_pci_id "$PORT_CFG") - VF_IDX=$(ovs_dpdk_get_vf_idx "$PORT_CFG") - REP=$(ovs_dpdk_get_representor_netdev $PF_PCI $VF_IDX) - AT_CHECK([test $? -eq 0]) + AT_CHECK([[[ "$VF_IDX" -ge 0 ]] && [[ "$VF_IDX" -le 5 ]] || return 66]) + + VF=$(ovs_get_vf_port $OVS_PF_PCI $VF_IDX) + AT_CHECK([test -n "$VF"]) - AT_CHECK([ip link set $REP name $1]) + AT_CHECK([ip link set $VF down]) + AT_CHECK([ip link set $VF name $1]) AT_CHECK([ip link set $1 netns $2]) AT_CHECK([ovs-vsctl add-port $3 ovs-$1 -- \ set interface ovs-$1 external-ids:iface-id="$1" -- \ set interface ovs-$1 type=dpdk -- \ set interface ovs-$1 \ - options:dpdk-devargs=$PF_PCI,representor=vf$VF_IDX]) + options:dpdk-devargs=$OVS_PF_PCI,representor=vf$VF_IDX]) NS_CHECK_EXEC([$2], [ip addr add $4 dev $1 $7]) NS_CHECK_EXEC([$2], [ip link set dev $1 up]) if test -n "$5"; then NS_CHECK_EXEC([$2], [ip link set dev $1 address $5]) else NS_CHECK_EXEC([$2], - [ip link set dev $1 address 02:00:00:00:EC:0$PORT_NO]) + [ip link set dev $1 address 02:00:00:00:EC:0$VF_IDX]) fi if test -n "$6"; then NS_CHECK_EXEC([$2], [ip route add default via $6]) fi on_exit "ip netns exec $2 ip link set $1 netns 1; \ - ip link property del dev $1 altname $REP; \ - ip link set $1 name $REP" + ip link property del dev $1 altname $VF; \ + ip link set $1 name $VF" ] ) m4_define([ADD_VETH], [ADD_VF($@)]) @@ -159,78 +156,3 @@ m4_define([DUMP_DP_IP_CLEAN_SORTED], [dnl grep 'eth_type(0x0800)' \ | sed -e 's/eth(src=[[a-z0-9:]]*,dst=[[a-z0-9:]]*)/eth(macs)/;s/recirc_id(0),//' \ | strip_used | strip_ptype | sort]) - -OVS_START_SHELL_HELPERS - -# ovs_dpdk_is_valid_pci_vf_addr() -# -# Check if the given PF PCI address and the VF number are valid. -# -ovs_dpdk_is_valid_pci_vf_addr() { - PCI_ID='[[0-9a-fA-F]]{4}:[[0-9a-fA-F]]{2}:[[0-9a-fA-F]]{2}\.[[0-7]]' - echo "$1" | grep -E -q "^$PCI_ID,[[0-9]]+$" && return 0 || return 1 -} - -# ovs_dpdk_get_pci_id() -# -ovs_dpdk_get_pci_id() { - printf '%s\n' "${1%%,*}" -} - -# ovs_dpdk_get_vf_idx() -# -ovs_dpdk_get_vf_idx() { - printf '%s\n' "${1##*,}" -} - -# ovs_dpdk_get_representor_netdev(<PF_PCI>, <VF_INDEX>) -# -# This function tries to find the representor netdev for the given PF's VF. -# -ovs_dpdk_get_representor_netdev() { - PF_PCI=$1 - VF_IDX=$2 - - VF_NET_DIR="/sys/bus/pci/devices/$PF_PCI/virtfn$VF_IDX/net" - - if [[ ! -d "$VF_NET_DIR" ]]; then - echo "ERROR: VF $VF_IDX for PF $PF_PCI does not exist" >&2 - return 1 - fi - - for iface in "$VF_NET_DIR"/*; do - if [[ -e "$iface" ]]; then - basename "$iface" - return 0 - fi - done - - echo "ERROR: No representor netdev found for VF $VF_IDX on PF $PF_PCI" >&2 - return 1 -} - -# ovs_dpdk_verify_vf_cfg() -# -# Verify that the given PF PCI addresses and corresponding VF IDs in -# OVS_DPDK_VF_PCI_ADDRS are valid, exist, and have corresponding -# representor ports. -# -ovs_dpdk_verify_vf_cfg() { - i=0 - - for addr in $1; do - ovs_dpdk_is_valid_pci_vf_addr "$addr" || return 1 - - PCI_ID=$(ovs_dpdk_get_pci_id "$addr") - VF_IDX=$(ovs_dpdk_get_vf_idx "$addr") - - REP=$(ovs_dpdk_get_representor_netdev $PCI_ID $VF_IDX) || return 1 - - echo "ovs-p$i: PF PCI $PCI_ID with VF $VF_IDX has representor $REP" - i=$((i + 1)) - done - - return 0 -} - -OVS_END_SHELL_HELPERS -- 2.47.3 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
