1. Add zxdh_recv_single_pkts fast path for single-segment RX. 2. Refactor scattered_rx decision and mirror it in zxdh_dev_mtu_set so MTU and scattered_rx stay consistent. 3. Remove unused xstats counters and dead struct fields, and fix data_off / hdr_size handling in the packed RX path.
Signed-off-by: Junlong Wang <[email protected]> --- drivers/net/zxdh/zxdh_ethdev.c | 40 +++++-- drivers/net/zxdh/zxdh_ethdev_ops.c | 23 ++-- drivers/net/zxdh/zxdh_ethdev_ops.h | 7 ++ drivers/net/zxdh/zxdh_queue.h | 21 ---- drivers/net/zxdh/zxdh_rxtx.c | 177 +++++++++++++++++++++++------ drivers/net/zxdh/zxdh_rxtx.h | 15 +-- 6 files changed, 198 insertions(+), 85 deletions(-) diff --git a/drivers/net/zxdh/zxdh_ethdev.c b/drivers/net/zxdh/zxdh_ethdev.c index a383619419..aa931b346a 100644 --- a/drivers/net/zxdh/zxdh_ethdev.c +++ b/drivers/net/zxdh/zxdh_ethdev.c @@ -70,8 +70,7 @@ zxdh_dev_infos_get(struct rte_eth_dev *dev, dev_info->reta_size = RTE_ETH_RSS_RETA_SIZE_256; dev_info->flow_type_rss_offloads = ZXDH_RSS_HF; - dev_info->max_mtu = ZXDH_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN - - RTE_VLAN_HLEN - ZXDH_DL_NET_HDR_SIZE; + dev_info->max_mtu = ZXDH_MAX_RX_PKTLEN - ZXDH_ETH_OVERHEAD - ZXDH_UL_NET_HDR_SIZE; dev_info->min_mtu = ZXDH_ETHER_MIN_MTU; dev_info->tx_offload_capa = (RTE_ETH_TX_OFFLOAD_MULTI_SEGS); @@ -1263,18 +1262,41 @@ zxdh_dev_close(struct rte_eth_dev *dev) return ret; } +/* + * Determine whether the current configuration requires support for scattered + * receive; return 1 if scattered receive is required and 0 if not. + */ +static bool +zxdh_scattered_rx(struct rte_eth_dev *eth_dev) +{ + uint16_t buf_size; + + if (eth_dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER) + return true; + + PMD_DRV_LOG(DEBUG, "port %u min_rx_buf_size %u", + eth_dev->data->port_id, eth_dev->data->min_rx_buf_size); + buf_size = eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; + if (eth_dev->data->mtu + ZXDH_ETH_OVERHEAD + ZXDH_UL_NET_HDR_SIZE > buf_size) + return true; + + return false; +} + static int32_t zxdh_set_rxtx_funcs(struct rte_eth_dev *eth_dev) { - struct zxdh_hw *hw = eth_dev->data->dev_private; - - if (!zxdh_pci_with_feature(hw, ZXDH_NET_F_MRG_RXBUF)) { - PMD_DRV_LOG(ERR, "port %u not support rx mergeable", eth_dev->data->port_id); - return -1; - } eth_dev->tx_pkt_prepare = zxdh_xmit_pkts_prepare; + eth_dev->data->scattered_rx = zxdh_scattered_rx(eth_dev); + eth_dev->data->lro = (eth_dev->data->dev_conf.rxmode.offloads & + RTE_ETH_RX_OFFLOAD_TCP_LRO) ? 1 : 0; + eth_dev->tx_pkt_burst = &zxdh_xmit_pkts_packed; - eth_dev->rx_pkt_burst = &zxdh_recv_pkts_packed; + + if (eth_dev->data->scattered_rx || eth_dev->data->lro) + eth_dev->rx_pkt_burst = &zxdh_recv_pkts_packed; + else + eth_dev->rx_pkt_burst = &zxdh_recv_single_pkts; return 0; } diff --git a/drivers/net/zxdh/zxdh_ethdev_ops.c b/drivers/net/zxdh/zxdh_ethdev_ops.c index 50247116d9..4929fa738d 100644 --- a/drivers/net/zxdh/zxdh_ethdev_ops.c +++ b/drivers/net/zxdh/zxdh_ethdev_ops.c @@ -95,10 +95,6 @@ static const struct rte_zxdh_xstats_name_off zxdh_rxq_stat_strings[] = { {"good_bytes", offsetof(struct zxdh_virtnet_rx, stats.bytes)}, {"errors", offsetof(struct zxdh_virtnet_rx, stats.errors)}, {"idle", offsetof(struct zxdh_virtnet_rx, stats.idle)}, - {"full", offsetof(struct zxdh_virtnet_rx, stats.full)}, - {"norefill", offsetof(struct zxdh_virtnet_rx, stats.norefill)}, - {"multicast_packets", offsetof(struct zxdh_virtnet_rx, stats.multicast)}, - {"broadcast_packets", offsetof(struct zxdh_virtnet_rx, stats.broadcast)}, {"truncated_err", offsetof(struct zxdh_virtnet_rx, stats.truncated_err)}, {"offload_cfg_err", offsetof(struct zxdh_virtnet_rx, stats.offload_cfg_err)}, {"invalid_hdr_len_err", offsetof(struct zxdh_virtnet_rx, stats.invalid_hdr_len_err)}, @@ -118,9 +114,6 @@ static const struct rte_zxdh_xstats_name_off zxdh_txq_stat_strings[] = { {"good_bytes", offsetof(struct zxdh_virtnet_tx, stats.bytes)}, {"errors", offsetof(struct zxdh_virtnet_tx, stats.errors)}, {"idle", offsetof(struct zxdh_virtnet_tx, stats.idle)}, - {"norefill", offsetof(struct zxdh_virtnet_tx, stats.norefill)}, - {"multicast_packets", offsetof(struct zxdh_virtnet_tx, stats.multicast)}, - {"broadcast_packets", offsetof(struct zxdh_virtnet_tx, stats.broadcast)}, {"truncated_err", offsetof(struct zxdh_virtnet_tx, stats.truncated_err)}, {"offload_cfg_err", offsetof(struct zxdh_virtnet_tx, stats.offload_cfg_err)}, {"invalid_hdr_len_err", offsetof(struct zxdh_virtnet_tx, stats.invalid_hdr_len_err)}, @@ -2026,6 +2019,22 @@ int zxdh_dev_mtu_set(struct rte_eth_dev *dev, uint16_t new_mtu) uint16_t vfid = zxdh_vport_to_vfid(hw->vport); int ret; + /* If device is started, refuse mtu that requires the support of + * scattered packets when this feature has not been enabled before. + */ + if (dev->data->dev_started) { + uint32_t buf_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; + uint8_t need_scatter = + (dev->data->dev_conf.rxmode.offloads & + (RTE_ETH_RX_OFFLOAD_TCP_LRO | RTE_ETH_RX_OFFLOAD_SCATTER)) || + (uint32_t)ZXDH_MTU_TO_PKTLEN(new_mtu) > buf_size; + + if (need_scatter != dev->data->scattered_rx) { + PMD_DRV_LOG(ERR, "Stop port first."); + return -EINVAL; + } + } + if (hw->is_pf) { ret = zxdh_get_panel_attr(dev, &panel); if (ret != 0) { diff --git a/drivers/net/zxdh/zxdh_ethdev_ops.h b/drivers/net/zxdh/zxdh_ethdev_ops.h index 6dfe4be473..ddd7d1578a 100644 --- a/drivers/net/zxdh/zxdh_ethdev_ops.h +++ b/drivers/net/zxdh/zxdh_ethdev_ops.h @@ -10,6 +10,7 @@ #include <rte_ether.h> #include "zxdh_ethdev.h" +#include "zxdh_queue.h" #define ZXDH_ETH_RSS_L2 RTE_ETH_RSS_L2_PAYLOAD #define ZXDH_ETH_RSS_IP \ @@ -40,6 +41,12 @@ #define ZXDH_SPM_SPEED_4X_100G RTE_BIT32(10) #define ZXDH_SPM_SPEED_4X_200G RTE_BIT32(11) +#define ZXDH_VLAN_TAG_LEN 4 +#define ZXDH_ETH_OVERHEAD (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + ZXDH_VLAN_TAG_LEN * 2) +/* Total per-packet fixed overhead: L2 wire overhead + uplink net header. */ +#define ZXDH_PKT_FIXED_OVERHEAD (ZXDH_ETH_OVERHEAD + ZXDH_UL_NET_HDR_SIZE) +#define ZXDH_MTU_TO_PKTLEN(mtu) ((mtu) + ZXDH_PKT_FIXED_OVERHEAD) + struct zxdh_np_stats_data { uint64_t n_pkts_dropped; uint64_t n_bytes_dropped; diff --git a/drivers/net/zxdh/zxdh_queue.h b/drivers/net/zxdh/zxdh_queue.h index 5bcc37cbce..562df88454 100644 --- a/drivers/net/zxdh/zxdh_queue.h +++ b/drivers/net/zxdh/zxdh_queue.h @@ -326,15 +326,6 @@ zxdh_queue_enable_intr(struct zxdh_virtqueue *vq) } } -static inline void -zxdh_mb(uint8_t weak_barriers) -{ - if (weak_barriers) - rte_atomic_thread_fence(rte_memory_order_seq_cst); - else - rte_mb(); -} - static inline int32_t desc_is_used(struct zxdh_vring_packed_desc *desc, struct zxdh_virtqueue *vq) { @@ -378,18 +369,6 @@ static inline void zxdh_queue_notify(struct zxdh_virtqueue *vq) rte_write32(notify_data, vq->notify_addr); } -static inline int32_t -zxdh_queue_kick_prepare_packed(struct zxdh_virtqueue *vq) -{ - uint16_t flags = 0; - - zxdh_mb(1); - flags = vq->vq_packed.ring.device->desc_event_flags; - - return (flags != ZXDH_RING_EVENT_FLAGS_DISABLE); -} - - struct rte_mbuf *zxdh_queue_detach_unused(struct zxdh_virtqueue *vq); int32_t zxdh_free_queues(struct rte_eth_dev *dev); int32_t zxdh_get_queue_type(uint16_t vtpci_queue_idx); diff --git a/drivers/net/zxdh/zxdh_rxtx.c b/drivers/net/zxdh/zxdh_rxtx.c index 93506a4b49..45fb2448a8 100644 --- a/drivers/net/zxdh/zxdh_rxtx.c +++ b/drivers/net/zxdh/zxdh_rxtx.c @@ -613,10 +613,12 @@ zxdh_dequeue_burst_rx_packed(struct zxdh_virtqueue *vq, uint16_t i, used_idx; uint16_t id; + used_idx = vq->vq_used_cons_idx; + rte_prefetch0(&desc[used_idx]); + for (i = 0; i < num; i++) { used_idx = vq->vq_used_cons_idx; - /** - * desc_is_used has a load-acquire or rte_io_rmb inside + /* desc_is_used has a load-acquire or rte_io_rmb inside * and wait for used desc in virtqueue. */ if (!desc_is_used(&desc[used_idx], vq)) @@ -823,17 +825,52 @@ zxdh_rx_update_mbuf(struct zxdh_hw *hw, struct rte_mbuf *m, struct zxdh_net_hdr_ } } -static void zxdh_discard_rxbuf(struct zxdh_virtqueue *vq, struct rte_mbuf *m) +static void +zxdh_refill_desc_unwrap(struct zxdh_virtqueue *vq, struct rte_mbuf **cookie, + uint16_t nb_pkts) { - int32_t error = 0; - /* - * Requeue the discarded mbuf. This should always be - * successful since it was just dequeued. - */ - error = zxdh_enqueue_recv_refill_packed(vq, &m, 1); - if (unlikely(error)) { - PMD_RX_LOG(ERR, "cannot enqueue discarded mbuf"); - rte_pktmbuf_free(m); + struct zxdh_vring_packed_desc *start_dp = vq->vq_packed.ring.desc; + struct zxdh_vq_desc_extra *dxp; + uint16_t flags = vq->cached_flags; + int32_t i; + uint16_t idx; + + idx = vq->vq_avail_idx; + for (i = 0; i < nb_pkts; i++) { + dxp = &vq->vq_descx[idx]; + dxp->cookie = (void *)cookie[i]; + start_dp[idx].addr = rte_mbuf_iova_get(cookie[i]) + RTE_PKTMBUF_HEADROOM; + start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM; + zxdh_queue_store_flags_packed(&start_dp[idx], flags); + idx++; + } + vq->vq_avail_idx += nb_pkts; + vq->vq_free_cnt = vq->vq_free_cnt - nb_pkts; +} + +static void +zxdh_refill_que_descs(struct zxdh_virtqueue *vq, struct rte_eth_dev *dev) +{ + /* free_cnt may include mrg descs */ + struct rte_mbuf *new_pkts[ZXDH_MBUF_BURST_SZ]; + uint16_t free_cnt = RTE_MIN(ZXDH_MBUF_BURST_SZ, vq->vq_free_cnt); + struct zxdh_virtnet_rx *rxvq = &vq->rxq; + uint16_t unwrap_cnt, left_cnt; + + if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { + left_cnt = free_cnt; + unwrap_cnt = 0; + if ((vq->vq_avail_idx + free_cnt) >= vq->vq_nentries) { + unwrap_cnt = vq->vq_nentries - vq->vq_avail_idx; + left_cnt = free_cnt - unwrap_cnt; + zxdh_refill_desc_unwrap(vq, new_pkts, unwrap_cnt); + vq->vq_avail_idx = 0; + vq->cached_flags ^= ZXDH_VRING_PACKED_DESC_F_AVAIL_USED; + } + if (left_cnt) + zxdh_refill_desc_unwrap(vq, new_pkts + unwrap_cnt, left_cnt); + } else { + dev->data->rx_mbuf_alloc_failed += free_cnt; } } @@ -852,7 +889,6 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t len = 0; uint32_t seg_num = 0; uint32_t seg_res = 0; - uint32_t error = 0; uint16_t hdr_size = 0; uint16_t nb_rx = 0; uint16_t i; @@ -873,7 +909,8 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rx_pkts[nb_rx] = rxm; prev = rxm; len = lens[i]; - header = rte_pktmbuf_mtod(rxm, struct zxdh_net_hdr_ul *); + header = (struct zxdh_net_hdr_ul *)((char *) + rxm->buf_addr + RTE_PKTMBUF_HEADROOM); seg_num = header->type_hdr.num_buffers; @@ -886,7 +923,7 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rxvq->stats.invalid_hdr_len_err++; continue; } - rxm->data_off += hdr_size; + rxm->data_off = RTE_PKTMBUF_HEADROOM + hdr_size; rxm->nb_segs = seg_num; rxm->ol_flags = 0; rcvd_pkt_len = len - hdr_size; @@ -902,18 +939,19 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, len = lens[i]; rxm = rcv_pkts[i]; rxm->data_len = len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; rcvd_pkt_len += len; prev->next = rxm; prev = rxm; rxm->next = NULL; - seg_res -= 1; + seg_res--; } if (!seg_res) { if (rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len) { PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d", rcvd_pkt_len, rx_pkts[nb_rx]->pkt_len); - zxdh_discard_rxbuf(vq, rx_pkts[nb_rx]); + rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; rxvq->stats.truncated_err++; continue; @@ -942,14 +980,14 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, prev->next = rxm; prev = rxm; rxm->next = NULL; - extra_idx += 1; + extra_idx++; } seg_res -= rcv_cnt; if (!seg_res) { if (unlikely(rcvd_pkt_len != rx_pkts[nb_rx]->pkt_len)) { PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d", rcvd_pkt_len, rx_pkts[nb_rx]->pkt_len); - zxdh_discard_rxbuf(vq, rx_pkts[nb_rx]); + rte_pktmbuf_free(rx_pkts[nb_rx]); rxvq->stats.errors++; rxvq->stats.truncated_err++; continue; @@ -961,26 +999,91 @@ zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, rxvq->stats.packets += nb_rx; refill: - /* Allocate new mbuf for the used descriptor */ - if (likely(!zxdh_queue_full(vq))) { - struct rte_mbuf *new_pkts[ZXDH_MBUF_BURST_SZ]; - /* free_cnt may include mrg descs */ - uint16_t free_cnt = RTE_MIN(vq->vq_free_cnt, ZXDH_MBUF_BURST_SZ); - - if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) { - error = zxdh_enqueue_recv_refill_packed(vq, new_pkts, free_cnt); - if (unlikely(error)) { - for (i = 0; i < free_cnt; i++) - rte_pktmbuf_free(new_pkts[i]); - } + if (vq->vq_free_cnt > 0) { + struct rte_eth_dev *dev = hw->eth_dev; + zxdh_refill_que_descs(vq, dev); + zxdh_queue_notify(vq); + } - if (unlikely(zxdh_queue_kick_prepare_packed(vq))) - zxdh_queue_notify(vq); - } else { - struct rte_eth_dev *dev = hw->eth_dev; + return nb_rx; +} - dev->data->rx_mbuf_alloc_failed += free_cnt; - } +static inline int +zxdh_init_mbuf(struct rte_mbuf *rxm, uint16_t len, + struct zxdh_hw *hw, struct zxdh_virtnet_rx *rxvq) +{ + uint16_t hdr_size = 0; + struct zxdh_net_hdr_ul *header; + + header = rte_pktmbuf_mtod(rxm, struct zxdh_net_hdr_ul *); + rxm->ol_flags = 0; + rxm->vlan_tci = 0; + rxm->vlan_tci_outer = 0; + + hdr_size = header->type_hdr.pd_len << 1; + if (unlikely(hdr_size > len || hdr_size < ZXDH_TYPE_HDR_SIZE || + header->type_hdr.num_buffers != 1)) { + PMD_RX_LOG(DEBUG, "hdr_size:%u nb_segs %d is invalid", + hdr_size, header->type_hdr.num_buffers); + rte_pktmbuf_free(rxm); + rxvq->stats.invalid_hdr_len_err++; + return -1; + } + zxdh_rx_update_mbuf(hw, rxm, header); + + rxm->nb_segs = 1; + rxm->data_off = RTE_PKTMBUF_HEADROOM + hdr_size; + rxm->data_len = len - hdr_size; + rxm->port = hw->port_id; + + if (rxm->data_len != rxm->pkt_len) { + PMD_RX_LOG(ERR, "dropped rcvd_pkt_len %d pktlen %d bufaddr %p.", + rxm->data_len, rxm->pkt_len, rxm->buf_addr); + rte_pktmbuf_free(rxm); + rxvq->stats.truncated_err++; + rxvq->stats.errors++; + return -1; + } + return 0; +} + +uint16_t +zxdh_recv_single_pkts(void *rx_queue, struct rte_mbuf **rcv_pkts, uint16_t nb_pkts) +{ + struct zxdh_virtnet_rx *rxvq = rx_queue; + struct zxdh_virtqueue *vq = rxvq->vq; + struct zxdh_hw *hw = vq->hw; + uint32_t lens[ZXDH_MBUF_BURST_SZ]; + uint16_t nb_rx = 0; + uint16_t num; + uint16_t i; + + num = nb_pkts; + if (unlikely(num > ZXDH_MBUF_BURST_SZ)) + num = ZXDH_MBUF_BURST_SZ; + num = zxdh_dequeue_burst_rx_packed(vq, rcv_pkts, lens, num); + if (num == 0) { + rxvq->stats.idle++; + goto refill; + } + + for (i = 0; i < num; i++) { + struct rte_mbuf *rxm = rcv_pkts[i]; + uint16_t len = lens[i]; + + if (unlikely(zxdh_init_mbuf(rxm, len, hw, &vq->rxq) < 0)) + continue; + rcv_pkts[nb_rx] = rxm; + zxdh_update_packet_stats(&rxvq->stats, rxm); + nb_rx++; + } + rxvq->stats.packets += nb_rx; + +refill: + if (vq->vq_free_cnt > 0) { + struct rte_eth_dev *dev = hw->eth_dev; + zxdh_refill_que_descs(vq, dev); + zxdh_queue_notify(vq); } return nb_rx; } diff --git a/drivers/net/zxdh/zxdh_rxtx.h b/drivers/net/zxdh/zxdh_rxtx.h index 424048607e..bf84c4796f 100644 --- a/drivers/net/zxdh/zxdh_rxtx.h +++ b/drivers/net/zxdh/zxdh_rxtx.h @@ -36,10 +36,6 @@ struct zxdh_virtnet_stats { uint64_t bytes; uint64_t errors; uint64_t idle; - uint64_t full; - uint64_t norefill; - uint64_t multicast; - uint64_t broadcast; uint64_t truncated_err; uint64_t offload_cfg_err; uint64_t invalid_hdr_len_err; @@ -49,16 +45,12 @@ struct zxdh_virtnet_stats { struct __rte_cache_aligned zxdh_virtnet_rx { struct zxdh_virtqueue *vq; - - uint64_t mbuf_initializer; /* value to init mbufs. */ struct rte_mempool *mpool; /* mempool for mbuf allocation */ - uint16_t queue_id; /* DPDK queue index. */ - uint16_t port_id; /* Device port identifier. */ struct zxdh_virtnet_stats stats; const struct rte_memzone *mz; /* mem zone to populate RX ring. */ - - /* dummy mbuf, for wraparound when processing RX ring. */ - struct rte_mbuf fake_mbuf; + uint64_t offloads; + uint16_t queue_id; /* DPDK queue index. */ + uint16_t port_id; /* Device port identifier. */ }; struct __rte_cache_aligned zxdh_virtnet_tx { @@ -75,5 +67,6 @@ struct __rte_cache_aligned zxdh_virtnet_tx { uint16_t zxdh_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t zxdh_xmit_pkts_prepare(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); uint16_t zxdh_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +uint16_t zxdh_recv_single_pkts(void *rx_queue, struct rte_mbuf **rcv_pkts, uint16_t nb_pkts); #endif /* ZXDH_RXTX_H */ -- 2.27.0
