Add a precomputed ptype table and a compact CQE Rx burst function, and select the Rx path from the rx_cqe_compact_en runtime parameter.
Signed-off-by: Jiacheng Ye <[email protected]> --- doc/guides/rel_notes/release_26_11.rst | 1 + drivers/net/hinic3/base/hinic3_nic_cfg.h | 2 + drivers/net/hinic3/hinic3_ethdev.c | 105 +++-- drivers/net/hinic3/hinic3_ethdev.h | 8 +- drivers/net/hinic3/hinic3_rx.c | 574 +++++++++++++++++++---- drivers/net/hinic3/hinic3_rx.h | 217 ++++----- 6 files changed, 661 insertions(+), 246 deletions(-) diff --git a/doc/guides/rel_notes/release_26_11.rst b/doc/guides/rel_notes/release_26_11.rst index d4cc028120..a8a1641d2a 100644 --- a/doc/guides/rel_notes/release_26_11.rst +++ b/doc/guides/rel_notes/release_26_11.rst @@ -100,6 +100,7 @@ New Features * Updated RSS hash type configuration. * Added support for the SP560 NIC. * Added device parameters for runtime configuration. + * Added a compact CQE receive data path. * **Updated Intel iavf driver.** diff --git a/drivers/net/hinic3/base/hinic3_nic_cfg.h b/drivers/net/hinic3/base/hinic3_nic_cfg.h index 0de8179c19..21ea74e44d 100644 --- a/drivers/net/hinic3/base/hinic3_nic_cfg.h +++ b/drivers/net/hinic3/base/hinic3_nic_cfg.h @@ -63,6 +63,8 @@ ((hinic3_get_driver_feature(dev) & NIC_F_##feature) != 0) #define HINIC3_SUPPORT_RX_HW_COMPACT_CQE(dev) \ HINIC3_SUPPORT_FEATURE(dev, RX_HW_COMPACT_CQE) +#define HINIC3_SUPPORT_RX_SW_COMPACT_CQE(dev) \ + HINIC3_SUPPORT_FEATURE(dev, RX_SW_COMPACT_CQE) #define HINIC3_SUPPORT_TX_WQE_COMPACT_TASK(dev) \ HINIC3_SUPPORT_FEATURE(dev, TX_WQE_COMPACT_TASK) #define HINIC3_SUPPORT_VXLAN_OFFLOAD(dev) \ diff --git a/drivers/net/hinic3/hinic3_ethdev.c b/drivers/net/hinic3/hinic3_ethdev.c index e5749ec718..b272d42007 100644 --- a/drivers/net/hinic3/hinic3_ethdev.c +++ b/drivers/net/hinic3/hinic3_ethdev.c @@ -1162,11 +1162,13 @@ hinic3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); - /* Queue depth must be equal to queue 0 */ - if (qid != 0 && nb_desc != nic_dev->rxqs[0]->q_depth) { - PMD_DRV_LOG(WARNING, "rxq%u depth:%u is not equal to queue0 depth:%u.", - qid, nb_desc, nic_dev->rxqs[0]->q_depth); - nb_desc = nic_dev->rxqs[0]->q_depth; + /* Queue depth must be equal to first queue */ + if (nic_dev->rxq_depth == 0) { + nic_dev->rxq_depth = nb_desc; + } else if (nb_desc != nic_dev->rxq_depth) { + PMD_DRV_LOG(WARNING, "rxq%u depth:%u is not equal to first queue depth:%u.", + qid, nb_desc, nic_dev->rxq_depth); + nb_desc = nic_dev->rxq_depth; } /* Queue depth must be power of 2, otherwise will be aligned up. */ @@ -1229,6 +1231,7 @@ hinic3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, rxq->port_id = dev->data->port_id; rxq->wait_time_cycle = HINIC3_RX_WAIT_CYCLE_THRESH; rxq->rx_deferred_start = rx_conf->rx_deferred_start; + rxq->rx_empty_loop = nic_dev->config.rx_empty_loop; /* If buf_len used for function table, need to translated. */ rx_buf_size = rte_pktmbuf_data_room_size(rxq->mb_pool) - RTE_PKTMBUF_HEADROOM; err = hinic3_convert_rx_buf_size(rx_buf_size, &buf_size); @@ -1239,7 +1242,7 @@ hinic3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, } /* If NIC support compact CQE, use compact wqe as default. */ - if (HINIC3_SUPPORT_RX_HW_COMPACT_CQE(nic_dev)) { + if (nic_dev->config.rx_cqe_compact_en) { rxq->wqe_type = HINIC3_COMPACT_RQ_WQE; } else { if (buf_size >= HINIC3_RX_BUF_SIZE_4K && @@ -1299,7 +1302,7 @@ hinic3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, goto alloc_rx_info_fail; } - if (HINIC3_SUPPORT_RX_HW_COMPACT_CQE(nic_dev)) { + if (nic_dev->config.rx_cqe_compact_en) { ci_mz = hinic3_dma_zone_reserve(dev, "hinic3_ci_mz", qid, sizeof(*rxq->rq_ci), RTE_CACHE_LINE_SIZE, (int)socket_id); @@ -1351,6 +1354,7 @@ hinic3_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qid, uint16_t nb_desc, hinic3_memzone_free(rxq->rq_mz); alloc_rq_mz_fail: + rxq->db_addr = NULL; alloc_db_err_fail: hinic3_memzone_free(rxq->pi_mz); @@ -1536,7 +1540,7 @@ hinic3_rx_queue_release(struct rte_eth_dev *dev, uint16_t queue_id) hinic3_free_rxq_mbufs(rxq); - if (HINIC3_SUPPORT_RX_HW_COMPACT_CQE(nic_dev)) + if (nic_dev->config.rx_cqe_compact_en) hinic3_memzone_free(rxq->ci_mz); else hinic3_memzone_free(rxq->cqe_mz); @@ -1699,6 +1703,9 @@ hinic3_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) if (!rte_intr_dp_is_en(intr_handle) || !intr_handle->intr_vec) return 0; + if (HINIC3_IS_SP230_NIC(nic_dev) && !dev->data->dev_conf.intr_conf.rxq) + return 0; + msix_intr = (uint16_t)intr_handle->intr_vec[queue_id]; hinic3_set_msix_auto_mask_state(nic_dev->hwdev, msix_intr, HINIC3_SET_MSIX_AUTO_MASK); @@ -2003,7 +2010,8 @@ hinic3_init_rxq_intr(struct rte_eth_dev *dev) intr_handle = dev->intr_handle; nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); - if (!dev->data->dev_conf.intr_conf.rxq) + + if (!HINIC3_IS_SP230_NIC(nic_dev) && !dev->data->dev_conf.intr_conf.rxq) return 0; if (!rte_intr_cap_multiple(intr_handle)) { @@ -2289,6 +2297,8 @@ hinic3_dev_stop(struct rte_eth_dev *dev) /* Free mempool. */ hinic3_copy_mempool_uninit(nic_dev); + + nic_dev->rxq_depth = 0; return 0; } @@ -2326,6 +2336,8 @@ hinic3_dev_release(struct rte_eth_dev *eth_dev) rte_free(nic_dev->hwdev); nic_dev->hwdev = NULL; + rte_free(nic_dev->ptype_tbl); + nic_dev->ptype_tbl = NULL; } /** @@ -3300,6 +3312,32 @@ hinic3_dev_xstats_get_names(struct rte_eth_dev *dev, return count; } +static const uint32_t * +hinic3_dev_supported_ptypes_get(__rte_unused struct rte_eth_dev *dev, size_t *no_of_elements) +{ + static const uint32_t ptypes[] = { + RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_SCTP, + RTE_PTYPE_L4_NONFRAG, + RTE_PTYPE_TUNNEL_IP, + RTE_PTYPE_TUNNEL_GRE, + RTE_PTYPE_TUNNEL_VXLAN, + RTE_PTYPE_TUNNEL_VXLAN_GPE, + RTE_PTYPE_TUNNEL_GENEVE, + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN, + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN, + RTE_PTYPE_INNER_L4_TCP, + RTE_PTYPE_INNER_L4_UDP, + RTE_PTYPE_INNER_L4_SCTP, + RTE_PTYPE_INNER_L4_NONFRAG, + }; + *no_of_elements = RTE_DIM(ptypes); + return ptypes; +} + static void hinic3_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo) @@ -3611,6 +3649,7 @@ hinic3_fec_capability_get(struct rte_eth_dev *dev, static const struct eth_dev_ops hinic3_pmd_ops = { .dev_configure = hinic3_dev_configure, .dev_infos_get = hinic3_dev_infos_get, + .dev_supported_ptypes_get = hinic3_dev_supported_ptypes_get, .fw_version_get = hinic3_fw_version_get, .dev_set_link_up = hinic3_dev_set_link_up, .dev_set_link_down = hinic3_dev_set_link_down, @@ -3662,6 +3701,7 @@ static const struct eth_dev_ops hinic3_pmd_ops = { static const struct eth_dev_ops hinic3_pmd_vf_ops = { .dev_configure = hinic3_dev_configure, .dev_infos_get = hinic3_dev_infos_get, + .dev_supported_ptypes_get = hinic3_dev_supported_ptypes_get, .fw_version_get = hinic3_fw_version_get, .dev_set_link_up = hinic3_dev_set_link_up, .dev_set_link_down = hinic3_dev_set_link_down, @@ -3713,16 +3753,6 @@ static void hinic3_nic_tx_rx_ops_init(struct hinic3_nic_dev *nic_dev) nic_dev->tx_ops->nic_tx_set_wqe_offload = hinic3_tx_set_compact_task_offload; else nic_dev->tx_ops->nic_tx_set_wqe_offload = hinic3_tx_set_normal_task_offload; - - if (HINIC3_SUPPORT_RX_HW_COMPACT_CQE(nic_dev)) { - nic_dev->rx_ops->nic_rx_get_cqe_info = hinic3_rx_get_compact_cqe_info; - nic_dev->rx_ops->nic_rx_cqe_done = hinic3_rx_integrated_cqe_done; - nic_dev->rx_ops->nic_rx_poll_rq_empty = hinic3_poll_integrated_cqe_rq_empty; - } else { - nic_dev->rx_ops->nic_rx_get_cqe_info = hinic3_rx_get_cqe_info; - nic_dev->rx_ops->nic_rx_cqe_done = hinic3_rx_separate_cqe_done; - nic_dev->rx_ops->nic_rx_poll_rq_empty = hinic3_poll_rq_empty; - } } /** @@ -3774,13 +3804,6 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) goto alloc_eth_addr_fail; } - nic_dev->rx_ops = rte_zmalloc("rx_ops", sizeof(struct hinic3_nic_rx_ops), 0); - if (!nic_dev->rx_ops) { - PMD_DRV_LOG(ERR, "Allocate rx_ops memory failed"); - err = -ENOMEM; - goto alloc_rx_ops_fail; - } - nic_dev->tx_ops = rte_zmalloc("tx_ops", sizeof(struct hinic3_nic_tx_ops), 0); if (!nic_dev->tx_ops) { PMD_DRV_LOG(ERR, "Allocate tx_ops memory failed"); @@ -3891,6 +3914,11 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) eth_dev->data->name); goto enable_intr_fail; } + + err = hinic3_init_rx_ptype_table(eth_dev); + if (err) + PMD_DRV_LOG(ERR, "Failed to create ptype_table."); + tcam_info = &nic_dev->tcam; memset(tcam_info, 0, sizeof(struct hinic3_tcam_info)); TAILQ_INIT(&tcam_info->tcam_list); @@ -3944,10 +3972,6 @@ hinic3_func_init(struct rte_eth_dev *eth_dev) nic_dev->tx_ops = NULL; alloc_tx_ops_fail: - rte_free(nic_dev->rx_ops); - nic_dev->rx_ops = NULL; - -alloc_rx_ops_fail: rte_free(eth_dev->data->mac_addrs); eth_dev->data->mac_addrs = NULL; @@ -4043,6 +4067,19 @@ hinic3_nic_common_config_get(struct rte_pci_device *pci_dev, return ret; } +static void hinic3_nic_feature_init(struct hinic3_nic_dev *nic_dev) +{ + if ((HINIC3_SUPPORT_RX_HW_COMPACT_CQE(nic_dev) || + HINIC3_SUPPORT_RX_SW_COMPACT_CQE(nic_dev)) && + nic_dev->config.rx_cqe_compact_en == HINIC3_RX_CQE_COMPACT_EN) { + nic_dev->config.rx_cqe_compact_en = HINIC3_RX_CQE_COMPACT_EN; + PMD_DRV_LOG(INFO, "nic compact_cqe is on."); + } else { + nic_dev->config.rx_cqe_compact_en = 0; + PMD_DRV_LOG(INFO, "nic compact_cqe is off."); + } +} + static int hinic3_dev_init(struct rte_eth_dev *eth_dev) { @@ -4072,7 +4109,13 @@ hinic3_dev_init(struct rte_eth_dev *eth_dev) return err; } - eth_dev->rx_pkt_burst = hinic3_recv_pkts; + hinic3_nic_feature_init(nic_dev); + + if (nic_dev->config.rx_cqe_compact_en != HINIC3_RX_CQE_COMPACT_EN) + eth_dev->rx_pkt_burst = hinic3_recv_pkts; + else + eth_dev->rx_pkt_burst = hinic3_recv_pkts_compact_cqe; + eth_dev->tx_pkt_burst = hinic3_xmit_pkts; return err; diff --git a/drivers/net/hinic3/hinic3_ethdev.h b/drivers/net/hinic3/hinic3_ethdev.h index 8191564fb8..41f6738645 100644 --- a/drivers/net/hinic3/hinic3_ethdev.h +++ b/drivers/net/hinic3/hinic3_ethdev.h @@ -117,6 +117,11 @@ enum nic_feature_cap { TAILQ_HEAD(hinic3_ethertype_filter_list, rte_flow); TAILQ_HEAD(hinic3_fdir_rule_filter_list, rte_flow); +#define HINIC3_PTYPE_NUM 4096 +struct hinic3_ptype_table { + alignas(RTE_CACHE_LINE_SIZE) uint32_t ptype[HINIC3_PTYPE_NUM]; +}; + struct hinic3_nic_common_dev_config { unsigned int rx_empty_threshold; /**< consecutive empty RX polls; skip RX if exceeded */ unsigned int rx_empty_loop; /**< max empty CQE spin count; 0 = no limit */ @@ -146,6 +151,7 @@ struct hinic3_nic_dev { uint16_t max_sqs; uint16_t max_rqs; + uint16_t rxq_depth; uint16_t rx_buff_len; uint16_t mtu_size; @@ -182,7 +188,7 @@ struct hinic3_nic_dev { struct hinic3_ethertype_filter_list filter_ethertype_list; struct hinic3_fdir_rule_filter_list filter_fdir_rule_list; uint8_t cos_map[HINIC3_COS_NUM_MAX]; - struct hinic3_nic_rx_ops *rx_ops; + struct hinic3_ptype_table *ptype_tbl; struct hinic3_nic_tx_ops *tx_ops; uint32_t fec_mode; /**< Current FEC mode for ethdev. */ enum nic_type nic_type; /**< NIC type. */ diff --git a/drivers/net/hinic3/hinic3_rx.c b/drivers/net/hinic3/hinic3_rx.c index 493855b431..fdf41a62ea 100644 --- a/drivers/net/hinic3/hinic3_rx.c +++ b/drivers/net/hinic3/hinic3_rx.c @@ -1,16 +1,248 @@ /* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2025 Huawei Technologies Co., Ltd */ +#include <rte_ether.h> +#include <rte_mbuf.h> #include "base/hinic3_compat.h" #include "base/hinic3_hwif.h" #include "base/hinic3_hwdev.h" #include "base/hinic3_wq.h" +#include "base/hinic3_mgmt.h" #include "base/hinic3_nic_cfg.h" #include "hinic3_nic_io.h" #include "hinic3_ethdev.h" +#include "hinic3_tx.h" #include "hinic3_rx.h" +static uint32_t +hinic3_get_l3_ptype(uint32_t ip_type) +{ + uint32_t ptype = 0; + if (ip_type == IPSU_METADATA_L3_TP_IPV4) + ptype |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ip_type == IPSU_METADATA_L3_TP_IPV6) + ptype |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + return ptype; +} +static uint32_t +hinic3_get_l4_ptype(uint32_t packet_type) +{ + int ptype = 0; + switch (packet_type) { + case IPSU_PKT_TYPE_NULL: + break; + case IPSU_PKT_TYPE_TCP: + ptype |= RTE_PTYPE_L4_TCP; + break; + case IPSU_PKT_TYPE_UDP: + ptype |= RTE_PTYPE_L4_UDP; + break; + case IPSU_PKT_TYPE_IPV4_FRAG: + ptype |= RTE_PTYPE_L4_FRAG; + break; + case IPSU_PKT_TYPE_SCTP: + ptype |= RTE_PTYPE_L4_SCTP; + break; + case IPSU_PKT_TYPE_ICMP: + ptype |= RTE_PTYPE_L4_ICMP; + break; + case IPSU_PKT_TYPE_IGMP: + ptype |= RTE_PTYPE_L4_IGMP; + break; + case IPSU_PKT_TYPE_ESP_OVER_IP: + ptype |= RTE_PTYPE_L4_ESP; + break; + default: + ptype |= RTE_PTYPE_L4_NONFRAG; + } + return ptype; +} + +/** + * Calculate the ptype value of given offload type. + * + * @param[in] offload_type + * The offload of Packet + * @return + * Ptype value + */ +static uint32_t +hinic3_calc_rx_ptype_table(uint32_t offload_type, struct hinic3_nic_dev *nic_dev) +{ + uint32_t packet_type, ip_type, enc_l3_type, pkt_fmt; + uint32_t ptype = RTE_PTYPE_UNKNOWN; + + packet_type = HINIC3_GET_RX_PKT_TYPE(offload_type); + ip_type = HINIC3_GET_RX_IP_TYPE(offload_type); + enc_l3_type = HINIC3_GET_RX_ENC_L3_TYPE(offload_type); + pkt_fmt = HINIC3_GET_RX_PKT_FORMAT(offload_type); + + switch (pkt_fmt) { + case IPSU_METADATA_FMT_NO_ENC: + ptype |= RTE_PTYPE_L2_ETHER; + ptype |= hinic3_get_l3_ptype(ip_type); + ptype |= hinic3_get_l4_ptype(packet_type); + break; + case IPSU_METADATA_FMT_FC: + ptype |= RTE_PTYPE_L2_ETHER_FCOE; + break; + case IPSU_METADATA_FMT_NSH: + ptype |= RTE_PTYPE_L2_ETHER_NSH; + break; + /* IPinIP */ + case IPSU_METADATA_FMT_IPIP: + ptype |= RTE_PTYPE_L2_ETHER; + ptype |= RTE_PTYPE_TUNNEL_IP; + if (!(nic_dev->feature_cap & NIC_F_TX_WQE_COMPACT_TASK)) { + ptype |= hinic3_get_l3_ptype(ip_type); + } else { + ptype |= hinic3_get_l3_ptype(enc_l3_type); + ptype |= hinic3_get_l4_ptype(packet_type) << HINIC3_L4_PYTPE_SHIFT; + if (ip_type == IPSU_METADATA_L3_TP_IPV4) + ptype |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; + else if (ip_type == IPSU_METADATA_L3_TP_IPV6) + ptype |= RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + } + break; + default: + /* VXLAN ãNVGRE ãVXLAN_GPE ãGENEVE*/ + ptype |= RTE_PTYPE_L2_ETHER; + ptype |= hinic3_get_l3_ptype(enc_l3_type); + + switch (pkt_fmt) { + case IPSU_METADATA_FMT_VXLAN: + ptype |= RTE_PTYPE_L4_UDP; + ptype |= RTE_PTYPE_TUNNEL_VXLAN; + ptype |= RTE_PTYPE_INNER_L2_ETHER; + break; + case IPSU_METADATA_FMT_NVGRE: + ptype |= RTE_PTYPE_TUNNEL_GRE; + if (!HINIC3_IS_SP560_NIC(nic_dev)) + return ptype; + break; + case IPSU_METADATA_FMT_GPE: + ptype |= RTE_PTYPE_L4_UDP; + ptype |= RTE_PTYPE_TUNNEL_VXLAN_GPE; + break; + case IPSU_METADATA_FMT_GENEVE: + ptype |= RTE_PTYPE_L4_UDP; + ptype |= RTE_PTYPE_TUNNEL_GENEVE; + ptype |= RTE_PTYPE_INNER_L2_ETHER; + break; + default: + ptype |= RTE_PTYPE_L4_NONFRAG; + break; + } + if (ip_type == IPSU_METADATA_L3_TP_IPV4) + ptype |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; + else if (ip_type == IPSU_METADATA_L3_TP_IPV6) + ptype |= RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + else + return ptype; + /* shift ptype to inner*/ + ptype |= hinic3_get_l4_ptype(packet_type) << HINIC3_L4_PYTPE_SHIFT; + } + return ptype; +} + +static uint32_t +hinic3_calc_rx_ptype_compact_table(uint32_t offload_type) +{ + uint32_t packet_type, ip_type, pkt_fmt; + uint32_t ptype = RTE_PTYPE_UNKNOWN; + + pkt_fmt = HINIC3_RQ_COMPACT_CQE_STATUS_GET(offload_type << HINIC3_COMPACT_CQE_PTYPE_SHIFT, + PKT_FORMAT); + ip_type = HINIC3_RQ_COMPACT_CQE_STATUS_GET(offload_type << HINIC3_COMPACT_CQE_PTYPE_SHIFT, + IP_TYPE); + packet_type = HINIC3_RQ_COMPACT_CQE_STATUS_GET(offload_type << + HINIC3_COMPACT_CQE_PTYPE_SHIFT, + PKT_TYPE); + + switch (pkt_fmt) { + case IPSU_METADATA_FMT_NO_ENC: + ptype |= RTE_PTYPE_L2_ETHER; + ptype |= hinic3_get_l3_ptype(ip_type); + ptype |= hinic3_get_l4_ptype(packet_type); + break; + case IPSU_METADATA_FMT_FC: + ptype |= RTE_PTYPE_L2_ETHER_FCOE; + break; + case IPSU_METADATA_FMT_NSH: + ptype |= RTE_PTYPE_L2_ETHER_NSH; + break; + default: + /* VXLAN ãNVGRE ãVXLAN_GPE ãGENEVE ãIPinIP */ + ptype |= RTE_PTYPE_L2_ETHER; + + switch (pkt_fmt) { + case IPSU_METADATA_FMT_VXLAN: + ptype |= RTE_PTYPE_L4_UDP; + ptype |= RTE_PTYPE_TUNNEL_VXLAN; + ptype |= RTE_PTYPE_INNER_L2_ETHER; + break; + case IPSU_METADATA_FMT_NVGRE: + ptype |= RTE_PTYPE_TUNNEL_GRE; + return ptype; + case IPSU_METADATA_FMT_GENEVE: + ptype |= RTE_PTYPE_L4_UDP; + ptype |= RTE_PTYPE_TUNNEL_GENEVE; + ptype |= RTE_PTYPE_INNER_L2_ETHER; + break; + + case IPSU_METADATA_FMT_IPIP: + ptype |= RTE_PTYPE_TUNNEL_IP; + break; + default: + ptype |= RTE_PTYPE_L4_NONFRAG; + break; + } + if (ip_type == IPSU_METADATA_L3_TP_IPV4) + ptype |= RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN; + else if (ip_type == IPSU_METADATA_L3_TP_IPV6) + ptype |= RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN; + else + return ptype; + /* shift ptype to inner*/ + ptype |= hinic3_get_l4_ptype(packet_type) << HINIC3_L4_PYTPE_SHIFT; + } + return ptype; +} + +/** + * Get receive queue local ci + * + * @param[in] rxq + * Receive queue + * @return + * Receive queue local ci + */ +static inline uint16_t hinic3_get_rq_local_ci(struct hinic3_rxq *rxq) +{ + return MASKED_QUEUE_IDX(rxq, rxq->cons_idx); +} + +static inline uint16_t hinic3_get_rq_free_wqebb(struct hinic3_rxq *rxq) +{ + return rxq->delta - 1; +} + +/** + * Update receive queue local ci + * + * @param[in] rxq + * Receive queue + * @param[in] wqe_cnt + * Wqebb counters + */ +static inline void hinic3_update_rq_local_ci(struct hinic3_rxq *rxq, + uint16_t wqe_cnt) +{ + rxq->cons_idx += wqe_cnt; + rxq->delta += wqe_cnt; +} + /** * Get wqe from receive queue. * @@ -62,6 +294,13 @@ hinic3_get_rq_local_pi(struct hinic3_rxq *rxq) return MASKED_QUEUE_IDX(rxq, rxq->prod_idx); } +static inline uint16_t hinic3_get_rq_hw_ci(struct hinic3_rxq *rxq) +{ + struct hinic3_rq_ci_wb rq_ci; + rq_ci.dw1.value = hinic3_hw_cpu32(rte_read32(&rxq->rq_ci->dw1.value)); + return rq_ci.dw1.bs.hw_ci; +} + uint16_t hinic3_rx_fill_wqe(struct hinic3_rxq *rxq) { @@ -434,11 +673,11 @@ hinic3_init_rss_type(struct hinic3_nic_dev *nic_dev, rss_type.ipv4 = (rss_hf & (RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4)) ? 1 : 0; rss_type.tcp_ipv4 = (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0; rss_type.ipv6 = (rss_hf & (RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6)) ? 1 : 0; + rss_type.ipv6_ext = (rss_hf & RTE_ETH_RSS_IPV6_EX) ? 1 : 0; rss_type.tcp_ipv6 = (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0; + rss_type.tcp_ipv6_ext = (rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) ? 1 : 0; rss_type.udp_ipv4 = (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0; rss_type.udp_ipv6 = (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0; - rss_type.ipv6_ext = (rss_hf & RTE_ETH_RSS_IPV6_EX) ? 1 : 0; - rss_type.tcp_ipv6_ext = (rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) ? 1 : 0; err = hinic3_set_rss_type(nic_dev->hwdev, rss_type); nic_dev->rss_type = rss_type; @@ -503,6 +742,36 @@ hinic3_update_rss_config(struct rte_eth_dev *dev, return err; } +/** + * Initialize the receive packet type table + * @param[in] dev + * Pointer to ethernet device structure. + */ + +int +hinic3_init_rx_ptype_table(struct rte_eth_dev *dev) +{ + struct hinic3_nic_dev *nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(dev); + struct hinic3_ptype_table *tbl = + rte_zmalloc("ptype_tbl", sizeof(struct hinic3_ptype_table), 0); + + if (tbl == NULL) + return -ENOMEM; + + uint32_t *ptype = tbl->ptype; + uint32_t i; + + if (nic_dev->config.rx_cqe_compact_en) { + for (i = 0; i < HINIC3_PTYPE_NUM; i++) + ptype[i] = hinic3_calc_rx_ptype_compact_table(i); + } else { + for (i = 0; i < HINIC3_PTYPE_NUM; i++) + ptype[i] = hinic3_calc_rx_ptype_table(i, nic_dev); + } + nic_dev->ptype_tbl = tbl; + return 0; +} + /** * Search given queue array to find position of given id. * Return queue pos or queue_count if not found. @@ -599,7 +868,7 @@ hinic3_rx_queue_release_mbufs(struct hinic3_rxq *rxq) hinic3_update_rq_local_ci(rxq, nr_released); } -int +static int hinic3_poll_rq_empty(struct hinic3_rxq *rxq) { uint64_t end; @@ -620,29 +889,21 @@ hinic3_poll_rq_empty(struct hinic3_rxq *rxq) return err; } -int +static int hinic3_poll_integrated_cqe_rq_empty(struct hinic3_rxq *rxq) { struct hinic3_rx_info *rx_info; - struct hinic3_rq_ci_wb rq_ci; - uint16_t sw_ci; - uint16_t hw_ci; - uint32_t val; + uint16_t sw_ci, sw_pi, hw_ci; + uint64_t end; sw_ci = hinic3_get_rq_local_ci(rxq); - val = rte_read32(&rxq->rq_ci->dw1.value); - rq_ci.dw1.value = hinic3_hw_cpu32(val); - hw_ci = rq_ci.dw1.bs.hw_ci; + sw_pi = hinic3_get_rq_local_pi(rxq); + hw_ci = hinic3_get_rq_hw_ci(rxq); if (HINIC3_IS_SP560_NIC(rxq->nic_dev)) { - uint16_t sw_pi = hinic3_get_rq_local_pi(rxq); - uint64_t end; - end = cycles + msecs_to_cycles(HINIC3_FLUSH_QUEUE_TIMEOUT); do { - val = rte_read32(&rxq->rq_ci->dw1.value); - rq_ci.dw1.value = hinic3_hw_cpu32(val); - hw_ci = rq_ci.dw1.bs.hw_ci; + hw_ci = hinic3_get_rq_hw_ci(rxq); if (sw_pi == hw_ci) break; @@ -666,22 +927,34 @@ hinic3_poll_integrated_cqe_rq_empty(struct hinic3_rxq *rxq) return 0; } -void -hinic3_dump_cqe_status(struct hinic3_rxq *rxq, uint32_t *cqe_done_cnt, - uint32_t *cqe_hole_cnt, uint32_t *head_ci, uint32_t *head_done) +static void +hinic3_dump_cqe_status(struct hinic3_rxq *rxq) { - uint16_t sw_ci; + volatile struct hinic3_rq_cqe *rx_cqe; + uint16_t sw_ci, sw_pi, hw_ci; + uint32_t head_ci, head_done; uint16_t avail_pkts = 0; uint16_t hit_done = 0; uint16_t cqe_hole = 0; - RTE_ATOMIC(uint32_t)status; - volatile struct hinic3_rq_cqe *rx_cqe; + uint32_t status; + + if (rxq->wqe_type == HINIC3_COMPACT_RQ_WQE) { + sw_ci = hinic3_get_rq_local_ci(rxq); + sw_pi = hinic3_get_rq_local_pi(rxq); + hw_ci = hinic3_get_rq_hw_ci(rxq); + PMD_DRV_LOG(ERR, + "Poll rq empty timeout, eth_dev:%s, queue_idx:%d, mbuf_left:%d, sw_pi:%d, sw_ci:%d, hw_ci:%d", + rxq->nic_dev->dev_name, rxq->q_id, + rxq->q_depth - hinic3_get_rq_free_wqebb(rxq), + sw_pi, sw_ci, hw_ci); + return; + } sw_ci = hinic3_get_rq_local_ci(rxq); rx_cqe = &rxq->rx_cqe[sw_ci]; status = rx_cqe->status; - *head_done = HINIC3_GET_RX_DONE(status); - *head_ci = sw_ci; + head_done = HINIC3_GET_RX_DONE(status); + head_ci = sw_ci; for (sw_ci = 0; sw_ci < rxq->q_depth; sw_ci++) { rx_cqe = &rxq->rx_cqe[sw_ci]; @@ -702,17 +975,17 @@ hinic3_dump_cqe_status(struct hinic3_rxq *rxq, uint32_t *cqe_done_cnt, hit_done = 1; } - *cqe_done_cnt = avail_pkts; - *cqe_hole_cnt = cqe_hole; + PMD_DRV_LOG(ERR, "Poll rq empty timeout, eth_dev:%s, queue_idx:%d, " + "mbuf_left:%d, cqe_done:%d, cqe_hole:%d, cqe[%d].done=%d", + rxq->nic_dev->dev_name, rxq->q_id, + rxq->q_depth - hinic3_get_rq_free_wqebb(rxq), + avail_pkts, cqe_hole, head_ci, head_done); } int hinic3_stop_rq(struct rte_eth_dev *eth_dev, struct hinic3_rxq *rxq) { struct hinic3_nic_dev *nic_dev = rxq->nic_dev; - uint32_t cqe_done_cnt = 0; - uint32_t cqe_hole_cnt = 0; - uint32_t head_ci, head_done; int err; /* Disable rxq intr. */ @@ -759,23 +1032,21 @@ hinic3_stop_rq(struct rte_eth_dev *eth_dev, struct hinic3_rxq *rxq) goto rq_flush_failed; } - err = nic_dev->rx_ops->nic_rx_poll_rq_empty(rxq); + if (nic_dev->config.rx_cqe_compact_en) + err = hinic3_poll_integrated_cqe_rq_empty(rxq); + else + err = hinic3_poll_rq_empty(rxq); + if (err) { - hinic3_dump_cqe_status(rxq, &cqe_done_cnt, &cqe_hole_cnt, - &head_ci, &head_done); - PMD_DRV_LOG(ERR, "Poll rq empty timeout"); - PMD_DRV_LOG(ERR, - "eth_dev:%s, queue_idx:%d, mbuf_left:%d, cqe_done:%d, cqe_hole:%d, cqe[%d].done=%d", - nic_dev->dev_name, rxq->q_id, - rxq->q_depth - hinic3_get_rq_free_wqebb(rxq), - cqe_done_cnt, cqe_hole_cnt, head_ci, head_done); + hinic3_dump_cqe_status(rxq); goto poll_rq_failed; } return 0; poll_rq_failed: - hinic3_set_rq_enable(nic_dev, rxq->q_id, true); + if ((hinic3_get_driver_feature(nic_dev) & NIC_F_HTN_FDIR) != 0) + (void)hinic3_set_rq_enable(nic_dev, rxq->q_id, true); rq_flush_failed: rte_spinlock_lock(&nic_dev->queue_list_lock); set_indir_failed: @@ -947,6 +1218,8 @@ hinic3_start_all_rqs(struct rte_eth_dev *eth_dev) for (i = 0; i < nic_dev->num_rqs; i++) { rxq = eth_dev->data->rx_queues[i]; + if (rxq == NULL) + break; hinic3_add_rq_to_rx_queue_list(nic_dev, rxq->q_id); err = hinic3_rearm_rxq_mbuf(rxq); if (err) { @@ -975,6 +1248,8 @@ hinic3_start_all_rqs(struct rte_eth_dev *eth_dev) out: for (i = 0; i < nic_dev->num_rqs; i++) { rxq = eth_dev->data->rx_queues[i]; + if (rxq == NULL) + continue; hinic3_remove_rq_from_rx_queue_list(nic_dev, rxq->q_id); hinic3_free_rxq_mbufs(rxq); hinic3_dev_rx_queue_intr_disable(eth_dev, rxq->q_id); @@ -982,38 +1257,14 @@ hinic3_start_all_rqs(struct rte_eth_dev *eth_dev) } return err; } - -bool -hinic3_rx_separate_cqe_done(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe **rx_cqe) -{ - volatile struct hinic3_rq_cqe *cqe = NULL; - uint16_t sw_ci; - uint32_t status; - - sw_ci = hinic3_get_rq_local_ci(rxq); - *rx_cqe = &rxq->rx_cqe[sw_ci]; - cqe = *rx_cqe; - - status = hinic3_hw_cpu32((uint32_t)(rte_atomic_load_explicit(&cqe->status, - rte_memory_order_acquire))); - if (!HINIC3_GET_RX_DONE(status)) - return false; - - return true; -} - -bool +static bool hinic3_rx_integrated_cqe_done(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe **rx_cqe) { - struct hinic3_rq_ci_wb rq_ci; struct rte_mbuf *rxm = NULL; uint16_t sw_ci, hw_ci; - uint32_t val; sw_ci = hinic3_get_rq_local_ci(rxq); - val = rte_read32(&rxq->rq_ci->dw1.value); - rq_ci.dw1.value = hinic3_hw_cpu32(val); - hw_ci = rq_ci.dw1.bs.hw_ci; + hw_ci = hinic3_get_rq_hw_ci(rxq); if (hw_ci == sw_ci) return false; @@ -1025,28 +1276,7 @@ hinic3_rx_integrated_cqe_done(struct hinic3_rxq *rxq, volatile struct hinic3_rq_ return true; } -void -hinic3_rx_get_cqe_info(struct hinic3_rxq *rxq __rte_unused, volatile struct hinic3_rq_cqe *rx_cqe, - struct hinic3_cqe_info *cqe_info) -{ - uint32_t dw0 = hinic3_hw_cpu32(rx_cqe->status); - uint32_t dw1 = hinic3_hw_cpu32(rx_cqe->vlan_len); - uint32_t dw2 = hinic3_hw_cpu32(rx_cqe->offload_type); - uint32_t dw3 = hinic3_hw_cpu32(rx_cqe->hash_val); - - cqe_info->lro_num = RQ_CQE_STATUS_GET(dw0, NUM_LRO); - cqe_info->csum_err = RQ_CQE_STATUS_GET(dw0, CSUM_ERR); - - cqe_info->pkt_len = RQ_CQE_SGE_GET(dw1, LEN); - cqe_info->vlan_tag = RQ_CQE_SGE_GET(dw1, VLAN); - - cqe_info->ptype = HINIC3_GET_RX_PTYPE_OFFLOAD(dw0); - cqe_info->vlan_offload = RQ_CQE_OFFOLAD_TYPE_GET(dw2, VLAN_EN); - cqe_info->rss_type = RQ_CQE_OFFOLAD_TYPE_GET(dw2, RSS_TYPE); - cqe_info->rss_hash_value = dw3; -} - -void +static void hinic3_rx_get_compact_cqe_info(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe *rx_cqe, struct hinic3_cqe_info *cqe_info) { @@ -1072,7 +1302,16 @@ hinic3_rx_get_compact_cqe_info(struct hinic3_rxq *rxq, volatile struct hinic3_rq cqe_info->ptype = HINIC3_RQ_COMPACT_CQE_STATUS_GET(dw0, PTYPE); cqe_info->rss_type = (rxq->nic_dev->rss_type.val != 0) ? 1 : 0; cqe_info->rss_hash_value = dw1; - + switch (cqe_info->csum_err) { + case HINIC3_RX_COMPACT_CSUM_OTHER_ERROR: + cqe_info->csum_err = HINIC3_RX_CSUM_IPSU_OTHER_ERR; + break; + case HINIC3_RX_COMPACT_HW_BYPASS_ERROR: + cqe_info->csum_err = HINIC3_RX_CSUM_HW_CHECK_NONE; + break; + default: + break; + } if (cqe_info->cqe_len == HINIC3_RQ_COMPACT_CQE_16BYTE) { cqe_info->lro_num = HINIC3_RQ_COMPACT_CQE_OFFLOAD_GET(dw2, NUM_LRO); cqe_info->vlan_tag = HINIC3_RQ_COMPACT_CQE_OFFLOAD_GET(dw2, VLAN); @@ -1083,12 +1322,12 @@ hinic3_rx_get_compact_cqe_info(struct hinic3_rxq *rxq, volatile struct hinic3_rq (cqe_info->cqe_len == HINIC3_RQ_COMPACT_CQE_16BYTE) ? 16 : 8; } -#define HINIC3_RX_EMPTY_THRESHOLD 3 uint16_t -hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +hinic3_recv_pkts_compact_cqe(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct hinic3_rxq *rxq = rx_queue; struct hinic3_nic_dev *nic_dev = rxq->nic_dev; + const struct hinic3_ptype_table * const ptype_tbl = nic_dev->ptype_tbl; struct hinic3_rx_info *rx_info = NULL; volatile struct hinic3_rq_cqe *rx_cqe = NULL; struct hinic3_cqe_info cqe_info = {0}; @@ -1102,19 +1341,18 @@ hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) uint64_t t2; #endif if (((rte_get_timer_cycles() - rxq->rxq_stats.tsc) < rxq->wait_time_cycle) && - rxq->rxq_stats.empty >= HINIC3_RX_EMPTY_THRESHOLD) + rxq->rxq_stats.empty >= nic_dev->config.rx_empty_threshold) goto out; sw_ci = hinic3_get_rq_local_ci(rxq); while (pkts < nb_pkts) { - rx_cqe = &rxq->rx_cqe[sw_ci]; - if (!nic_dev->rx_ops->nic_rx_cqe_done(rxq, &rx_cqe)) { + if (!hinic3_rx_integrated_cqe_done(rxq, &rx_cqe)) { rxq->rxq_stats.empty++; break; } - nic_dev->rx_ops->nic_rx_get_cqe_info(rxq, rx_cqe, &cqe_info); + hinic3_rx_get_compact_cqe_info(rxq, rx_cqe, &cqe_info); pkt_len = cqe_info.pkt_len; /* @@ -1160,8 +1398,9 @@ hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) /* 5. Vlan offload. */ rxm->ol_flags |= hinic3_rx_vlan(cqe_info.vlan_offload, cqe_info.vlan_tag, &rxm->vlan_tci); - - /* 6. RSS. */ + /* 6. Packet ptype */ + rxm->packet_type = ptype_tbl->ptype[cqe_info.ptype]; + /* 7. RSS. */ rxm->ol_flags |= hinic3_rx_rss_hash(cqe_info.rss_type, cqe_info.rss_hash_value, &rxm->hash.rss); /* 8. LRO. */ @@ -1201,3 +1440,144 @@ hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) return pkts; } + +uint16_t hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) +{ + struct hinic3_rxq *rxq = rx_queue; + struct hinic3_nic_dev *nic_dev = rxq->nic_dev; + struct hinic3_rx_info *rx_info = NULL; + volatile struct hinic3_rq_cqe *rx_cqe = NULL; + struct rte_mbuf *rxm = NULL; + uint16_t sw_ci, rx_buf_len, wqebb_cnt = 0, pkts = 0; + uint32_t status, pkt_len, vlan_len, offload_type, pkt_type, lro_num; + uint64_t rx_bytes = 0; + uint32_t hash_value = 0; + uint32_t empty_nb = 0; + const struct hinic3_ptype_table * const ptype_tbl = nic_dev->ptype_tbl; + +#ifdef HINIC3_XSTAT_PROF_RX + uint64_t t1 = rte_get_tsc_cycles(); + uint64_t t2; +#endif + if (((rte_get_timer_cycles() - rxq->rxq_stats.tsc) < rxq->wait_time_cycle) && + rxq->rxq_stats.empty >= nic_dev->config.rx_empty_threshold) + goto out; + + sw_ci = hinic3_get_rq_local_ci(rxq); + rx_buf_len = rxq->buf_len; + + while (pkts < nb_pkts) { + rx_cqe = &rxq->rx_cqe[sw_ci]; + status = hinic3_hw_cpu32((uint32_t)(rte_atomic_load_explicit(&rx_cqe->status, + rte_memory_order_acquire))); + if (!HINIC3_GET_RX_DONE(status)) { + rxq->rxq_stats.empty++; + if (empty_nb++ >= rxq->rx_empty_loop) + break; + continue; + } + + vlan_len = hinic3_hw_cpu32(rx_cqe->vlan_len); + + pkt_len = HINIC3_GET_RX_PKT_LEN(vlan_len); + + rx_info = &rxq->rx_info[sw_ci]; + rxm = rx_info->mbuf; + + /* 1. Next ci point and prefetch */ + sw_ci++; + sw_ci &= rxq->q_mask; + + /* 2. Prefetch next mbuf first 64B */ + rte_prefetch0(rxq->rx_info[sw_ci].mbuf); + + /* 3. Jumbo frame process */ + + if (likely(pkt_len <= (uint32_t)rx_buf_len)) { + rxm->data_len = (uint16_t)pkt_len; + rxm->pkt_len = pkt_len; + wqebb_cnt++; + } else { + rxm->data_len = rx_buf_len; + rxm->pkt_len = rx_buf_len; + + /* If receive jumbo, updating ci will be done by + * hinic3_recv_jumbo_pkt function. + */ + hinic3_update_rq_local_ci(rxq, wqebb_cnt + 1); + wqebb_cnt = 0; + hinic3_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len); + sw_ci = hinic3_get_rq_local_ci(rxq); + } + + + /* + * data_off was already set by rearm to the offset of the DMA + * start address relative to buf (HEADROOM + alignment offset + * when alignment is enabled, HEADROOM when disabled). The packet + * data starts at buf + data_off, so no further adjustment is + * needed. + */ + rxm->port = rxq->port_id; + + /* 4. Rx checksum offload */ + rxm->ol_flags |= hinic3_rx_csum(HINIC3_GET_RX_CSUM_ERR(status), rxq); + + /* 5. Vlan offload */ + offload_type = hinic3_hw_cpu32(rx_cqe->offload_type); + + rxm->ol_flags |= hinic3_rx_vlan(HINIC3_GET_RX_VLAN_OFFLOAD_EN(offload_type), + HINIC3_GET_RX_VLAN_TAG(vlan_len), + &rxm->vlan_tci); + + /* 6. Packet ptype */ + pkt_type = HINIC3_GET_RX_PTYPE_OFFLOAD(offload_type); + rxm->packet_type = ptype_tbl->ptype[pkt_type]; + + /* 7. RSS */ + hash_value = hinic3_hw_cpu32(rx_cqe->hash_val); + rxm->ol_flags |= hinic3_rx_rss_hash(HINIC3_GET_RSS_TYPES(offload_type), + hash_value, + &rxm->hash.rss); + + /* 8. LRO */ + lro_num = HINIC3_GET_RX_NUM_LRO(status); + if (unlikely(lro_num != 0)) { + rxm->ol_flags |= HINIC3_PKT_RX_LRO; + rxm->tso_segsz = pkt_len / lro_num; + } + + rx_cqe->status = 0; + + rx_bytes += pkt_len; + rx_pkts[pkts++] = rxm; + } + + if (pkts) { + /* 9. Update local ci */ + hinic3_update_rq_local_ci(rxq, wqebb_cnt); + + /* Update packet stats */ + rxq->rxq_stats.packets += pkts; + rxq->rxq_stats.bytes += rx_bytes; + rxq->rxq_stats.empty = 0; +#ifdef HINIC3_XSTAT_MBUF_USE + rxq->rxq_stats.free_mbuf += pkts; +#endif + } + rxq->rxq_stats.burst_pkts = pkts; + rxq->rxq_stats.tsc = rte_get_timer_cycles(); +out: + /* 10. Rearm mbuf to rxq */ + hinic3_rearm_rxq_mbuf(rxq); + +#ifdef HINIC3_XSTAT_PROF_RX + /* do profiling stats */ + t2 = rte_get_tsc_cycles(); + rxq->rxq_stats.app_tsc = t1 - rxq->prof_rx_end_tsc; + rxq->prof_rx_end_tsc = t2; + rxq->rxq_stats.pmd_tsc = t2 - t1; +#endif + + return pkts; +} diff --git a/drivers/net/hinic3/hinic3_rx.h b/drivers/net/hinic3/hinic3_rx.h index 129c2b4a59..0cdfcb6e9c 100644 --- a/drivers/net/hinic3/hinic3_rx.h +++ b/drivers/net/hinic3/hinic3_rx.h @@ -5,11 +5,23 @@ #ifndef _HINIC3_RX_H_ #define _HINIC3_RX_H_ +#include "hinic3_wq.h" + #define RQ_CQE_OFFOLAD_TYPE_PTYPE_OFFLOAD_SHIFT 0 +#define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_SHIFT 0 +#define RQ_CQE_OFFOLAD_TYPE_IP_TYPE_SHIFT 5 +#define RQ_CQE_OFFOLAD_TYPE_ENC_L3_TYPE_SHIFT 7 +#define RQ_CQE_OFFOLAD_TYPE_PKT_FORMAT_SHIFT 8 +#define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_SHIFT 19 #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_SHIFT 21 #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_SHIFT 24 #define RQ_CQE_OFFOLAD_TYPE_PTYPE_OFFLOAD_MASK 0xFFFU +#define RQ_CQE_OFFOLAD_TYPE_PKT_TYPE_MASK 0x1FU +#define RQ_CQE_OFFOLAD_TYPE_IP_TYPE_MASK 0x3U +#define RQ_CQE_OFFOLAD_TYPE_ENC_L3_TYPE_MASK 0X1U +#define RQ_CQE_OFFOLAD_TYPE_PKT_FORMAT_MASK 0xFU +#define RQ_CQE_OFFOLAD_TYPE_PKT_UMBCAST_MASK 0x3U #define RQ_CQE_OFFOLAD_TYPE_VLAN_EN_MASK 0x1U #define RQ_CQE_OFFOLAD_TYPE_RSS_TYPE_MASK 0xFFU @@ -25,6 +37,15 @@ #define HINIC3_GET_RX_PKT_TYPE(offload_type) \ RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_TYPE) +#define HINIC3_GET_RX_IP_TYPE(offload_type) \ + RQ_CQE_OFFOLAD_TYPE_GET(offload_type, IP_TYPE) + +#define HINIC3_GET_RX_ENC_L3_TYPE(offload_type) \ + RQ_CQE_OFFOLAD_TYPE_GET(offload_type, ENC_L3_TYPE) + +#define HINIC3_GET_RX_PKT_FORMAT(offload_type) \ + RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_FORMAT) + #define HINIC3_GET_RX_PKT_UMBCAST(offload_type) \ RQ_CQE_OFFOLAD_TYPE_GET(offload_type, PKT_UMBCAST) @@ -142,10 +163,10 @@ #define RQ_COMPACT_CQE_STATUS_CQE_TYPE_MASK 0x1U #define RQ_COMPACT_CQE_STATUS_TS_FLAG_MASK 0x1U #define RQ_COMPACT_CQE_STATUS_VLAN_EN_MASK 0x1U -#define RQ_COMPACT_CQE_STATUS_PKT_FORMAT_MASK 0x7U +#define RQ_COMPACT_CQE_STATUS_PKT_FORMAT_MASK 0x7U #define RQ_COMPACT_CQE_STATUS_IP_TYPE_MASK 0x1U #define RQ_COMPACT_CQE_STATUS_CQE_LEN_MASK 0x1U -#define RQ_COMPACT_CQE_STATUS_PKT_MC_MASK 0x1U +#define RQ_COMPACT_CQE_STATUS_PKT_MC_MASK 0x3U #define RQ_COMPACT_CQE_STATUS_CSUM_ERR_MASK 0x3U #define RQ_COMPACT_CQE_STATUS_PKT_TYPE_MASK 0x7U #define RQ_COMPACT_CQE_STATUS_PTYPE_MASK 0xFFFU @@ -182,6 +203,13 @@ #define HINIC3_RX_CSUM_HW_CHECK_NONE RTE_BIT32(7) #define HINIC3_RX_CSUM_IPSU_OTHER_ERR RTE_BIT32(8) +enum hinic3_compact_cqe_csum_err_type { + HINIC3_RX_COMPACT_CSUM_NO_ERROR = 0, + HINIC3_RX_COMPACT_L3_L4_CSUM_ERROR, + HINIC3_RX_COMPACT_CSUM_OTHER_ERROR, + HINIC3_RX_COMPACT_HW_BYPASS_ERROR +}; + #define HINIC3_DEFAULT_RX_CSUM_OFFLOAD 0xFFF #define HINIC3_CQE_LEN 32 @@ -200,6 +228,72 @@ RTE_ETH_RSS_IPV6_TCP_EX | \ RTE_ETH_RSS_IPV6_UDP_EX) +#define HINIC3_L4_PYTPE_SHIFT 16 +#define HINIC3_COMPACT_CQE_PTYPE_SHIFT 16 + +/* keep same with IPSU_METADATA_L3_TP_E */ +enum HINIC3_RX_CQE_PT_L3 { + HINIC3_RX_CQE_L3_IPV4 = 0u, + HINIC3_RX_CQE_L3_IPV6 = 1u, +}; + +/* keep same with IPSU_PKT_TYPE_L45FINAL_E */ +enum HINIC3_RX_CQE_PT_L4 { + HINIC3_RX_CQE_L4_TCP = 3, + HINIC3_RX_CQE_L4_UDP = 4, +}; + +enum IPSU_METADATA_L3_TP_E { + IPSU_METADATA_L3_TP_IPV4 = 0u, + IPSU_METADATA_L3_TP_IPV6 = 1u, +}; + +enum IPSU_PKT_TYPE_L45FINAL_E { + IPSU_PKT_TYPE_NULL = 0, + IPSU_PKT_TYPE_ROCEV2, + IPSU_PKT_TYPE_TCPCOCO, + IPSU_PKT_TYPE_TCP, + IPSU_PKT_TYPE_UDP, + IPSU_PKT_TYPE_ICMP, + IPSU_PKT_TYPE_IGMP, + IPSU_PKT_TYPE_SCTP, + IPSU_PKT_TYPE_DHCP, + IPSU_PKT_TYPE_IPV4_FRAG, + IPSU_PKT_TYPE_IPV6_MC, + IPSU_PKT_TYPE_1588, + IPSU_PKT_TYPE_AH_OVER_IP, + IPSU_PKT_TYPE_ESP_OVER_IP, + IPSU_PKT_TYPE_NATT, + IPSU_PKT_TYPE_AH_OVER_VXLAN_GPE, + IPSU_PKT_TYPE_ESP_OVER_VXLAN_GPE, + IPSU_PKT_TYPE_TSO, + IPSU_PKT_TYPE_UFO, + IPSU_PKT_TYPE_INT, + IPSU_PKT_TYPE_IOAM, + IPSU_PKT_TYPE_VXLAN_GPE, + IPSU_PKT_TYPE_GENEVE, + IPSU_PKT_TYPE_NSH_OVER_GENEVE, + IPSU_PKT_TYPE_NSH_OVER_VXLAN_GPE, + IPSU_PKT_TYPE_ESP_OVER_GENEVE, + IPSU_PKT_TYPE_PPOP_OVER_GENEVE, + IPSU_PKT_TYPE_OSPF, + IPSU_PKT_TYPE_VRRP, + IPSU_PKT_TYPE_BGP, + IPSU_PKT_TYPE_GRE, + IPSU_ERR_RSVD, +}; + +enum IPSU_METADATA_FMT_E { + IPSU_METADATA_FMT_NO_ENC = 0u, + IPSU_METADATA_FMT_VXLAN = 1u, + IPSU_METADATA_FMT_NVGRE = 2u, + IPSU_METADATA_FMT_FC = 3u, + IPSU_METADATA_FMT_GPE = 4u, + IPSU_METADATA_FMT_GENEVE = 5u, + IPSU_METADATA_FMT_NSH = 6u, + IPSU_METADATA_FMT_IPIP = 7U, +}; + struct hinic3_rxq_stats { uint64_t packets; uint64_t bytes; @@ -313,7 +407,6 @@ struct hinic3_rq_ci_wb { uint32_t rsvd[3]; }; - struct __rte_cache_aligned hinic3_rxq { struct hinic3_nic_dev *nic_dev; @@ -323,6 +416,7 @@ struct __rte_cache_aligned hinic3_rxq { uint16_t buf_len; uint32_t rx_buff_shift; + uint32_t rx_empty_loop; uint16_t rx_free_thresh; uint16_t rxinfo_align_end; @@ -370,24 +464,6 @@ struct __rte_cache_aligned hinic3_rxq { #endif }; -/* Rx CQE info get callback function */ -typedef void (*nic_rx_get_cqe_info_t)(struct hinic3_rxq *rxq, - volatile struct hinic3_rq_cqe *rx_cqe, - struct hinic3_cqe_info *cqe_info); - -/* Rx CQE check status callback function */ -typedef bool (*nic_rx_cqe_done_t)(struct hinic3_rxq *rxq, - volatile struct hinic3_rq_cqe **rx_cqe); - -/* Rx CQE empty poll callback function */ -typedef int (*nic_rx_poll_rq_empty_t)(struct hinic3_rxq *rxq); - -struct hinic3_nic_rx_ops { - nic_rx_get_cqe_info_t nic_rx_get_cqe_info; - nic_rx_cqe_done_t nic_rx_cqe_done; - nic_rx_poll_rq_empty_t nic_rx_poll_rq_empty; -}; - uint16_t hinic3_rx_fill_wqe(struct hinic3_rxq *rxq); uint16_t hinic3_rx_fill_buffers(struct hinic3_rxq *rxq); @@ -399,16 +475,14 @@ void hinic3_free_all_rxq_mbufs(struct hinic3_nic_dev *nic_dev); int hinic3_update_rss_config(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); -int hinic3_poll_integrated_cqe_rq_empty(struct hinic3_rxq *rxq); -int hinic3_poll_rq_empty(struct hinic3_rxq *rxq); - -void hinic3_dump_cqe_status(struct hinic3_rxq *rxq, uint32_t *cqe_done_cnt, - uint32_t *cqe_hole_cnt, uint32_t *head_ci, uint32_t *head_done); +int hinic3_init_rx_ptype_table(struct rte_eth_dev *dev); int hinic3_stop_rq(struct rte_eth_dev *eth_dev, struct hinic3_rxq *rxq); int hinic3_start_rq(struct rte_eth_dev *eth_dev, struct hinic3_rxq *rxq); +uint16_t hinic3_recv_pkts_compact_cqe(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); + uint16_t hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); void hinic3_add_rq_to_rx_queue_list(struct hinic3_nic_dev *nic_dev, @@ -426,95 +500,4 @@ int hinic3_start_all_rqs(struct rte_eth_dev *eth_dev); void hinic3_get_stats(struct hinic3_rxq *rxq); #endif -/** - * Get receive queue local ci. - * - * @param[in] rxq - * Pointer to receive queue structure. - * @return - * Receive queue local ci. - */ -static inline uint16_t -hinic3_get_rq_local_ci(struct hinic3_rxq *rxq) -{ - return MASKED_QUEUE_IDX(rxq, rxq->cons_idx); -} - -static inline uint16_t -hinic3_get_rq_free_wqebb(struct hinic3_rxq *rxq) -{ - return rxq->delta - 1; -} - -/** - * Update receive queue local ci. - * - * @param[in] rxq - * Pointer to receive queue structure. - * @param[out] wqe_cnt - * Wqebb counters. - */ -static inline void -hinic3_update_rq_local_ci(struct hinic3_rxq *rxq, uint16_t wqe_cnt) -{ - rxq->cons_idx += wqe_cnt; - rxq->delta += wqe_cnt; -} - -/** - * Get receive cqe information - * - * @param[in] rxq - * Receive queue - * @param[in] rx_cqe - * Receive cqe - * @param[in] cqe_info - * Packet information parsed from cqe - */ -void -hinic3_rx_get_cqe_info(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe *rx_cqe, - struct hinic3_cqe_info *cqe_info); - -/** - * Get receive compact cqe information - * - * @param[in] rx_queue - * Receive queue - * @param[in] rx_cqe - * Receive compact cqe - * @param[in] cqe_info - * Packet information parsed from cqe - */ -void -hinic3_rx_get_compact_cqe_info(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe *rx_cqe, - struct hinic3_cqe_info *cqe_info); - -/** - * Check whether pkt is received when CQE is separated - * - * @param[in] rxq - * Receive queue - * @param[in] rx_cqe - * The CQE written by hw - * @return - * True: Packet is received - * False: Packet is not received - */ -bool -hinic3_rx_separate_cqe_done(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe **rx_cqe); - -/** - * Check whether pkt is received when CQE is integrated - * - * @param[in] rxq - * Receive queue - * @param[in] rx_cqe - * The CQE written by hw - * @return - * True: Packet is received - * False: Packet is not received - */ -bool -hinic3_rx_integrated_cqe_done(struct hinic3_rxq *rxq, volatile struct hinic3_rq_cqe **rx_cqe); - #endif /* _HINIC3_RX_H_ */ -- 2.33.0
