Adds new fastpath routines for cn10k & cn9k endpoint
devices and assigns the fastpath routines based on
the offload flags.
Patch also adds misc changes to improve performance
and code-readability.
Signed-off-by: Vamsi Attunuru <vattun...@marvell.com>
---
drivers/net/octeon_ep/cnxk_ep_rx.c | 309 ++++++++++++++++++++++++++
drivers/net/octeon_ep/cnxk_ep_tx.c | 209 +++++++++++++++++
drivers/net/octeon_ep/cnxk_ep_vf.c | 2 +
drivers/net/octeon_ep/cnxk_ep_vf.h | 13 ++
drivers/net/octeon_ep/meson.build | 2 +
drivers/net/octeon_ep/otx2_ep_vf.c | 1 +
drivers/net/octeon_ep/otx_ep_common.h | 125 ++++++-----
drivers/net/octeon_ep/otx_ep_ethdev.c | 69 +++++-
drivers/net/octeon_ep/otx_ep_rxtx.c | 93 +-------
drivers/net/octeon_ep/otx_ep_rxtx.h | 38 +++-
10 files changed, 704 insertions(+), 157 deletions(-)
diff --git a/drivers/net/octeon_ep/cnxk_ep_rx.c
b/drivers/net/octeon_ep/cnxk_ep_rx.c
new file mode 100644
index 0000000000..74f0011283
--- /dev/null
+++ b/drivers/net/octeon_ep/cnxk_ep_rx.c
@@ -0,0 +1,309 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#include "otx_ep_common.h"
+#include "otx2_ep_vf.h"
+#include "otx_ep_rxtx.h"
+
+static inline int
+cnxk_ep_rx_refill_mbuf(struct otx_ep_droq *droq, uint32_t count)
+{
+ struct otx_ep_droq_desc *desc_ring = droq->desc_ring;
+ struct rte_mbuf **recv_buf_list = droq->recv_buf_list;
+ uint32_t refill_idx = droq->refill_idx;
+ struct rte_mbuf *buf;
+ uint32_t i;
+ int rc;
+
+ rc = rte_pktmbuf_alloc_bulk(droq->mpool, &recv_buf_list[refill_idx],
count);
+ if (unlikely(rc)) {
+ droq->stats.rx_alloc_failure++;
+ return rc;
+ }
+
+ for (i = 0; i < count; i++) {
+ buf = recv_buf_list[refill_idx];
+ desc_ring[refill_idx].buffer_ptr =
rte_mbuf_data_iova_default(buf);
+ refill_idx++;
+ }
+
+ droq->refill_idx = otx_ep_incr_index(droq->refill_idx, count,
droq->nb_desc);
+ droq->refill_count -= count;
+
+ return 0;
+}
+
+static inline void
+cnxk_ep_rx_refill(struct otx_ep_droq *droq)
+{
+ uint32_t desc_refilled = 0, count;
+ uint32_t nb_desc = droq->nb_desc;
+ uint32_t refill_idx = droq->refill_idx;
+ int rc;
+
+ if (unlikely(droq->read_idx == refill_idx))
+ return;
+
+ if (refill_idx < droq->read_idx) {
+ count = droq->read_idx - refill_idx;
+ rc = cnxk_ep_rx_refill_mbuf(droq, count);
+ if (unlikely(rc)) {
+ droq->stats.rx_alloc_failure++;
+ return;
+ }
+ desc_refilled = count;
+ } else {
+ count = nb_desc - refill_idx;
+ rc = cnxk_ep_rx_refill_mbuf(droq, count);
+ if (unlikely(rc)) {
+ droq->stats.rx_alloc_failure++;
+ return;
+ }
+
+ desc_refilled = count;
+ count = droq->read_idx;
+ rc = cnxk_ep_rx_refill_mbuf(droq, count);
+ if (unlikely(rc)) {
+ droq->stats.rx_alloc_failure++;
+ return;
+ }
+ desc_refilled += count;
+ }
+
+ /* Flush the droq descriptor data to memory to be sure
+ * that when we update the credits the data in memory is
+ * accurate.
+ */
+ rte_io_wmb();
+ rte_write32(desc_refilled, droq->pkts_credit_reg);
+}
+
+static inline uint32_t
+cnxk_ep_check_rx_pkts(struct otx_ep_droq *droq)
+{
+ uint32_t new_pkts;
+ uint32_t val;
+
+ /* Batch subtractions from the HW counter to reduce PCIe traffic
+ * This adds an extra local variable, but almost halves the
+ * number of PCIe writes.
+ */
+ val = __atomic_load_n(droq->pkts_sent_ism, __ATOMIC_RELAXED);
+ new_pkts = val - droq->pkts_sent_ism_prev;
+ droq->pkts_sent_ism_prev = val;
+
+ if (val > (uint32_t)(1 << 31)) {
+ /* Only subtract the packet count in the HW counter
+ * when count above halfway to saturation.
+ */
+ rte_write64((uint64_t)val, droq->pkts_sent_reg);
+ rte_mb();
+
+ rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg);
+ while (__atomic_load_n(droq->pkts_sent_ism, __ATOMIC_RELAXED)
>= val) {
+ rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg);
+ rte_mb();
+ }
+
+ droq->pkts_sent_ism_prev = 0;
+ }
+ rte_write64(OTX2_SDP_REQUEST_ISM, droq->pkts_sent_reg);
+ droq->pkts_pending += new_pkts;
+
+ return new_pkts;
+}
+
+static inline int16_t __rte_hot
+cnxk_ep_rx_pkts_to_process(struct otx_ep_droq *droq, uint16_t nb_pkts)
+{
+ if (droq->pkts_pending < nb_pkts)
+ cnxk_ep_check_rx_pkts(droq);
+
+ return RTE_MIN(nb_pkts, droq->pkts_pending);
+}
+
+static __rte_always_inline void
+cnxk_ep_process_pkts_scalar(struct rte_mbuf **rx_pkts, struct otx_ep_droq
*droq, uint16_t new_pkts)
+{
+ struct rte_mbuf **recv_buf_list = droq->recv_buf_list;
+ uint32_t bytes_rsvd = 0, read_idx = droq->read_idx;
+ uint16_t port_id = droq->otx_ep_dev->port_id;
+ uint16_t nb_desc = droq->nb_desc;
+ uint16_t pkts;
+
+ for (pkts = 0; pkts < new_pkts; pkts++) {
+ struct otx_ep_droq_info *info;
+ struct rte_mbuf *mbuf;
+ uint16_t pkt_len;
+
+ mbuf = recv_buf_list[read_idx];
+ info = rte_pktmbuf_mtod(mbuf, struct otx_ep_droq_info *);
+ read_idx = otx_ep_incr_index(read_idx, 1, nb_desc);
+ pkt_len = rte_bswap16(info->length >> 48);
+ mbuf->data_off += OTX_EP_INFO_SIZE;
+ mbuf->pkt_len = pkt_len;
+ mbuf->data_len = pkt_len;
+ mbuf->port = port_id;
+ rx_pkts[pkts] = mbuf;
+ bytes_rsvd += pkt_len;
+ }
+ droq->read_idx = read_idx;
+
+ droq->refill_count += new_pkts;
+ droq->pkts_pending -= new_pkts;
+ /* Stats */
+ droq->stats.pkts_received += new_pkts;
+ droq->stats.bytes_received += bytes_rsvd;
+}
+
+static __rte_always_inline void
+cnxk_ep_process_pkts_scalar_mseg(struct rte_mbuf **rx_pkts, struct otx_ep_droq
*droq,
+ uint16_t new_pkts)
+{
+ struct rte_mbuf **recv_buf_list = droq->recv_buf_list;
+ uint32_t total_pkt_len, bytes_rsvd = 0;
+ uint16_t port_id = droq->otx_ep_dev->port_id;
+ uint16_t nb_desc = droq->nb_desc;
+ uint16_t pkts;
+
+ for (pkts = 0; pkts < new_pkts; pkts++) {
+ struct otx_ep_droq_info *info;
+ struct rte_mbuf *first_buf = NULL;
+ struct rte_mbuf *last_buf = NULL;
+ struct rte_mbuf *mbuf;
+ uint32_t pkt_len = 0;
+
+ mbuf = recv_buf_list[droq->read_idx];
+ info = rte_pktmbuf_mtod(mbuf, struct otx_ep_droq_info *);
+
+ total_pkt_len = rte_bswap16(info->length >> 48) +
OTX_EP_INFO_SIZE;
+
+ while (pkt_len < total_pkt_len) {
+ int cpy_len;
+
+ cpy_len = ((pkt_len + droq->buffer_size) >
total_pkt_len)
+ ? ((uint32_t)total_pkt_len - pkt_len) :
droq->buffer_size;
+
+ mbuf = droq->recv_buf_list[droq->read_idx];
+
+ if (!pkt_len) {
+ /* Note the first seg */
+ first_buf = mbuf;
+ mbuf->data_off += OTX_EP_INFO_SIZE;
+ mbuf->pkt_len = cpy_len - OTX_EP_INFO_SIZE;
+ mbuf->data_len = cpy_len - OTX_EP_INFO_SIZE;
+ } else {
+ mbuf->pkt_len = cpy_len;
+ mbuf->data_len = cpy_len;
+ first_buf->nb_segs++;
+ first_buf->pkt_len += mbuf->pkt_len;
+ }
+
+ if (last_buf)
+ last_buf->next = mbuf;
+
+ last_buf = mbuf;
+
+ pkt_len += cpy_len;
+ droq->read_idx = otx_ep_incr_index(droq->read_idx, 1,
nb_desc);
+ droq->refill_count++;
+ }
+ mbuf = first_buf;
+ mbuf->port = port_id;
+ rx_pkts[pkts] = mbuf;
+ bytes_rsvd += pkt_len;
+ }
+
+ droq->refill_count += new_pkts;
+ droq->pkts_pending -= pkts;
+ /* Stats */
+ droq->stats.pkts_received += pkts;
+ droq->stats.bytes_received += bytes_rsvd;
+}
+
+uint16_t __rte_noinline __rte_hot
+cnxk_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
+ uint16_t new_pkts;
+
+ new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
+ cnxk_ep_process_pkts_scalar(rx_pkts, droq, new_pkts);
+
+ /* Refill RX buffers */
+ if (droq->refill_count >= DROQ_REFILL_THRESHOLD)
+ cnxk_ep_rx_refill(droq);
+
+ return new_pkts;
+}
+
+uint16_t __rte_noinline __rte_hot
+cn9k_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
+{
+ struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
+ uint16_t new_pkts;
+
+ new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
+ cnxk_ep_process_pkts_scalar(rx_pkts, droq, new_pkts);
+
+ /* Refill RX buffers */
+ if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
+ cnxk_ep_rx_refill(droq);
+ } else {
+ /* SDP output goes into DROP state when output doorbell count
+ * goes below drop count. When door bell count is written with
+ * a value greater than drop count SDP output should come out
+ * of DROP state. Due to a race condition this is not happening.
+ * Writing doorbell register with 0 again may make SDP output
+ * come out of this state.
+ */
+
+ rte_write32(0, droq->pkts_credit_reg);
+ }
+
+ return new_pkts;
+}
+
+uint16_t __rte_noinline __rte_hot
+cnxk_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t
nb_pkts)
+{
+ struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
+ uint16_t new_pkts;
+
+ new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
+ cnxk_ep_process_pkts_scalar_mseg(rx_pkts, droq, new_pkts);
+
+ /* Refill RX buffers */
+ if (droq->refill_count >= DROQ_REFILL_THRESHOLD)
+ cnxk_ep_rx_refill(droq);
+
+ return new_pkts;
+}
+
+uint16_t __rte_noinline __rte_hot
+cn9k_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t
nb_pkts)
+{
+ struct otx_ep_droq *droq = (struct otx_ep_droq *)rx_queue;
+ uint16_t new_pkts;
+
+ new_pkts = cnxk_ep_rx_pkts_to_process(droq, nb_pkts);
+ cnxk_ep_process_pkts_scalar_mseg(rx_pkts, droq, new_pkts);
+
+ /* Refill RX buffers */
+ if (droq->refill_count >= DROQ_REFILL_THRESHOLD) {
+ cnxk_ep_rx_refill(droq);
+ } else {
+ /* SDP output goes into DROP state when output doorbell count
+ * goes below drop count. When door bell count is written with
+ * a value greater than drop count SDP output should come out
+ * of DROP state. Due to a race condition this is not happening.
+ * Writing doorbell register with 0 again may make SDP output
+ * come out of this state.
+ */
+
+ rte_write32(0, droq->pkts_credit_reg);
+ }
+
+ return new_pkts;
+}
diff --git a/drivers/net/octeon_ep/cnxk_ep_tx.c
b/drivers/net/octeon_ep/cnxk_ep_tx.c
new file mode 100644
index 0000000000..9f11a2f317
--- /dev/null
+++ b/drivers/net/octeon_ep/cnxk_ep_tx.c
@@ -0,0 +1,209 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2023 Marvell.
+ */
+
+#include "cnxk_ep_vf.h"
+#include "otx_ep_rxtx.h"
+
+static uint32_t
+cnxk_vf_update_read_index(struct otx_ep_instr_queue *iq)
+{
+ uint32_t val;
+
+ /* Batch subtractions from the HW counter to reduce PCIe traffic
+ * This adds an extra local variable, but almost halves the
+ * number of PCIe writes.
+ */
+ val = __atomic_load_n(iq->inst_cnt_ism, __ATOMIC_RELAXED);
+ iq->inst_cnt += val - iq->inst_cnt_ism_prev;
+ iq->inst_cnt_ism_prev = val;
+
+ if (val > (uint32_t)(1 << 31)) {
+ /* Only subtract the packet count in the HW counter
+ * when count above halfway to saturation.
+ */
+ rte_write64((uint64_t)val, iq->inst_cnt_reg);
+ rte_mb();
+
+ rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg);
+ while (__atomic_load_n(iq->inst_cnt_ism, __ATOMIC_RELAXED) >=
val) {
+ rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg);
+ rte_mb();
+ }
+
+ iq->inst_cnt_ism_prev = 0;
+ }
+ rte_write64(OTX2_SDP_REQUEST_ISM, iq->inst_cnt_reg);
+
+ /* Modulo of the new index with the IQ size will give us
+ * the new index.
+ */
+ return iq->inst_cnt & (iq->nb_desc - 1);
+}
+
+static inline void
+cnxk_ep_flush_iq(struct otx_ep_instr_queue *iq)
+{
+ uint32_t instr_processed = 0;
+ uint32_t cnt = 0;
+
+ iq->otx_read_index = cnxk_vf_update_read_index(iq);
+
+ if (unlikely(iq->flush_index == iq->otx_read_index))
+ return;
+
+ if (iq->flush_index < iq->otx_read_index) {
+ instr_processed = iq->otx_read_index - iq->flush_index;
+ rte_pktmbuf_free_bulk(&iq->mbuf_list[iq->flush_index],
instr_processed);
+ iq->flush_index = otx_ep_incr_index(iq->flush_index,
instr_processed, iq->nb_desc);
+ } else {
+ cnt = iq->nb_desc - iq->flush_index;
+ rte_pktmbuf_free_bulk(&iq->mbuf_list[iq->flush_index], cnt);
+ iq->flush_index = otx_ep_incr_index(iq->flush_index, cnt,
iq->nb_desc);
+
+ instr_processed = iq->otx_read_index;
+ rte_pktmbuf_free_bulk(&iq->mbuf_list[iq->flush_index],
instr_processed);
+ iq->flush_index = otx_ep_incr_index(iq->flush_index,
instr_processed, iq->nb_desc);
+
+ instr_processed += cnt;
+ }
+
+ iq->stats.instr_processed = instr_processed;
+ iq->instr_pending -= instr_processed;
+}
+
+static inline void
+set_sg_size(struct otx_ep_sg_entry *sg_entry, uint16_t size, uint32_t pos)
+{
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ sg_entry->u.size[pos] = size;
+#elif RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
+ sg_entry->u.size[(OTX_EP_NUM_SG_PTRS - 1) - pos] = size;
+#endif
+}
+
+static __rte_always_inline void
+cnxk_ep_xmit_pkts_scalar(struct rte_mbuf **tx_pkts, struct otx_ep_instr_queue
*iq, uint16_t nb_pkts)
+{
+ struct cnxk_ep_instr_32B *iqcmd;
+ struct rte_mbuf *m;
+ uint32_t pkt_len;
+ uint32_t tx_bytes = 0;
+ uint32_t write_idx = iq->host_write_index;
+ uint16_t pkts, nb_desc = iq->nb_desc;
+ uint8_t desc_size = iq->desc_size;
+
+ for (pkts = 0; pkts < nb_pkts; pkts++) {
+ m = tx_pkts[pkts];
+ iq->mbuf_list[write_idx] = m;
+ pkt_len = rte_pktmbuf_data_len(m);
+
+ iqcmd = (struct cnxk_ep_instr_32B *)(iq->base_addr + (write_idx
* desc_size));
+ iqcmd->ih.u64 = iq->partial_ih | pkt_len;
+ iqcmd->dptr = rte_mbuf_data_iova(m); /*dptr*/
+ tx_bytes += pkt_len;
+
+ /* Increment the host write index */
+ write_idx = otx_ep_incr_index(write_idx, 1, nb_desc);
+ }
+ iq->host_write_index = write_idx;
+
+ /* ring dbell */
+ rte_io_wmb();
+ rte_write64(pkts, iq->doorbell_reg);
+ iq->instr_pending += pkts;
+ iq->stats.tx_pkts += pkts;
+ iq->stats.tx_bytes += tx_bytes;
+}
+
+static __rte_always_inline uint16_t
+cnxk_ep_xmit_pkts_scalar_mseg(struct rte_mbuf **tx_pkts, struct
otx_ep_instr_queue *iq,
+ uint16_t nb_pkts)
+{
+ uint16_t frags, num_sg, mask = OTX_EP_NUM_SG_PTRS - 1;
+ struct otx_ep_buf_free_info *finfo;
+ struct cnxk_ep_instr_32B *iqcmd;
+ struct rte_mbuf *m;
+ uint32_t pkt_len, tx_bytes = 0;
+ uint32_t write_idx = iq->host_write_index;
+ uint16_t pkts, nb_desc = iq->nb_desc;
+ uint8_t desc_size = iq->desc_size;
+
+ for (pkts = 0; pkts < nb_pkts; pkts++) {
+ uint16_t j = 0;
+
+ m = tx_pkts[pkts];
+ frags = m->nb_segs;
+
+ pkt_len = rte_pktmbuf_pkt_len(m);
+ num_sg = (frags + mask) / OTX_EP_NUM_SG_PTRS;
+
+ if (unlikely(pkt_len > OTX_EP_MAX_PKT_SZ && num_sg >
OTX_EP_MAX_SG_LISTS)) {
+ otx_ep_err("Failed to xmit the pkt, pkt_len is higher
or pkt has more segments\n");
+ goto exit;
+ }
+
+ finfo = &iq->req_list[write_idx].finfo;
+
+ iq->mbuf_list[write_idx] = m;
+ iqcmd = (struct cnxk_ep_instr_32B *)(iq->base_addr + (write_idx
* desc_size));
+ iqcmd->dptr = rte_mem_virt2iova(finfo->g.sg);
+ iqcmd->ih.u64 = iq->partial_ih | (1ULL << 62) |
((uint64_t)frags << 48) | pkt_len;
+
+ while (frags--) {
+ finfo->g.sg[(j >> 2)].ptr[(j & mask)] =
rte_mbuf_data_iova(m);
+ set_sg_size(&finfo->g.sg[(j >> 2)], m->data_len, (j &
mask));
+ j++;
+ m = m->next;
+ }
+
+ /* Increment the host write index */
+ write_idx = otx_ep_incr_index(write_idx, 1, nb_desc);
+ tx_bytes += pkt_len;
+ }
+exit:
+ iq->host_write_index = write_idx;
+
+ /* ring dbell */
+ rte_io_wmb();
+ rte_write64(pkts, iq->doorbell_reg);
+ iq->instr_pending += pkts;
+ iq->stats.tx_pkts += pkts;
+ iq->stats.tx_bytes += tx_bytes;
+
+ return pkts;
+}
+
+uint16_t __rte_noinline __rte_hot
+cnxk_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
+{
+ struct otx_ep_instr_queue *iq = (struct otx_ep_instr_queue *)tx_queue;
+ uint16_t pkts;
+
+ pkts = RTE_MIN(nb_pkts, iq->nb_desc - iq->instr_pending);
+
+ cnxk_ep_xmit_pkts_scalar(tx_pkts, iq, pkts);
+
+ if (iq->instr_pending >= OTX_EP_MAX_INSTR)
+ cnxk_ep_flush_iq(iq);
+
+ /* Return no# of instructions posted successfully. */
+ return pkts;
+}
+
+uint16_t __rte_noinline __rte_hot
+cnxk_ep_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t
nb_pkts)
+{
+ struct otx_ep_instr_queue *iq = (struct otx_ep_instr_queue *)tx_queue;
+ uint16_t pkts;
+
+ pkts = RTE_MIN(nb_pkts, iq->nb_desc - iq->instr_pending);
+
+ pkts = cnxk_ep_xmit_pkts_scalar_mseg(tx_pkts, iq, pkts);
+
+ if (iq->instr_pending >= OTX_EP_MAX_INSTR)
+ cnxk_ep_flush_iq(iq);
+
+ /* Return no# of instructions posted successfully. */
+ return pkts;
+}
diff --git a/drivers/net/octeon_ep/cnxk_ep_vf.c
b/drivers/net/octeon_ep/cnxk_ep_vf.c
index 7b3669fe0c..ef275703c3 100644
--- a/drivers/net/octeon_ep/cnxk_ep_vf.c
+++ b/drivers/net/octeon_ep/cnxk_ep_vf.c
@@ -156,6 +156,8 @@ cnxk_ep_vf_setup_iq_regs(struct otx_ep_device *otx_ep,
uint32_t iq_no)
(void *)iq->inst_cnt_ism, ism_addr);
*iq->inst_cnt_ism = 0;
iq->inst_cnt_ism_prev = 0;
+ iq->partial_ih = ((uint64_t)otx_ep->pkind) << 36;
+
return 0;
}
diff --git a/drivers/net/octeon_ep/cnxk_ep_vf.h
b/drivers/net/octeon_ep/cnxk_ep_vf.h
index 86277449ea..41d8fbbb3a 100644
--- a/drivers/net/octeon_ep/cnxk_ep_vf.h
+++ b/drivers/net/octeon_ep/cnxk_ep_vf.h
@@ -6,6 +6,8 @@
#include <rte_io.h>
+#include "otx_ep_common.h"
+
#define CNXK_CONFIG_XPANSION_BAR 0x38
#define CNXK_CONFIG_PCIE_CAP 0x70
#define CNXK_CONFIG_PCIE_DEVCAP 0x74
@@ -178,6 +180,17 @@ struct cnxk_ep_instr_64B {
uint64_t exhdr[4];
};
+struct cnxk_ep_instr_32B {
+ /* Pointer where the input data is available. */
+ uint64_t dptr;
+
+ /* OTX_EP Instruction Header. */
+ union otx_ep_instr_ih ih;
+
+ /* Misc data bytes that can be passed as front data */
+ uint64_t rsvd[2];
+};
+
#define CNXK_EP_IQ_ISM_OFFSET(queue) (RTE_CACHE_LINE_SIZE * (queue) + 4)
#define CNXK_EP_OQ_ISM_OFFSET(queue) (RTE_CACHE_LINE_SIZE * (queue))
#define CNXK_EP_ISM_EN (0x1)
diff --git a/drivers/net/octeon_ep/meson.build
b/drivers/net/octeon_ep/meson.build
index e698bf9792..749776d70c 100644
--- a/drivers/net/octeon_ep/meson.build
+++ b/drivers/net/octeon_ep/meson.build
@@ -9,4 +9,6 @@ sources = files(
'otx2_ep_vf.c',
'cnxk_ep_vf.c',
'otx_ep_mbox.c',
+ 'cnxk_ep_rx.c',
+ 'cnxk_ep_tx.c',
)
diff --git a/drivers/net/octeon_ep/otx2_ep_vf.c
b/drivers/net/octeon_ep/otx2_ep_vf.c
index f72b8d25d7..7f4edf8dcf 100644
--- a/drivers/net/octeon_ep/otx2_ep_vf.c
+++ b/drivers/net/octeon_ep/otx2_ep_vf.c
@@ -307,6 +307,7 @@ otx2_vf_setup_iq_regs(struct otx_ep_device *otx_ep,
uint32_t iq_no)
(unsigned int)ism_addr);
*iq->inst_cnt_ism = 0;
iq->inst_cnt_ism_prev = 0;
+ iq->partial_ih = ((uint64_t)otx_ep->pkind) << 36;
return 0;
}
diff --git a/drivers/net/octeon_ep/otx_ep_common.h
b/drivers/net/octeon_ep/otx_ep_common.h
index 90e059cad0..82e57520d3 100644
--- a/drivers/net/octeon_ep/otx_ep_common.h
+++ b/drivers/net/octeon_ep/otx_ep_common.h
@@ -4,7 +4,20 @@
#ifndef _OTX_EP_COMMON_H_
#define _OTX_EP_COMMON_H_
+#include <rte_bitops.h>
#include <rte_spinlock.h>
+#include <unistd.h>
+#include <assert.h>
+#include <rte_eal.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_io.h>
+#include <rte_net.h>
+#include <ethdev_pci.h>
+
+#define OTX_EP_CN8XX RTE_BIT32(0)
+#define OTX_EP_CN9XX RTE_BIT32(1)
+#define OTX_EP_CN10XX RTE_BIT32(2)
#define OTX_EP_NW_PKT_OP 0x1220
#define OTX_EP_NW_CMD_OP 0x1221
@@ -38,7 +51,7 @@
#define OTX_EP_NORESP_OHSM_SEND (4)
#define OTX_EP_NORESP_LAST (4)
#define OTX_EP_PCI_RING_ALIGN 65536
-#define OTX_EP_MAX_SG_LISTS 4
+#define OTX_EP_MAX_SG_LISTS 6
#define OTX_EP_NUM_SG_PTRS 4
#define SDP_PKIND 40
#define SDP_OTX2_PKIND 57
@@ -203,6 +216,38 @@ struct otx_ep_iq_config {
* such structure to represent it.
*/
struct otx_ep_instr_queue {
+ /* Location in memory updated by SDP ISM */
+ uint32_t *inst_cnt_ism;
+ struct rte_mbuf **mbuf_list;
+ /* Pointer to the Virtual Base addr of the input ring. */
+ uint8_t *base_addr;
+
+ /* track inst count locally to consolidate HW counter updates */
+ uint32_t inst_cnt_ism_prev;
+
+ /* Input ring index, where the driver should write the next packet */
+ uint32_t host_write_index;
+
+ /* Input ring index, where the OCTEON 9 should read the next packet */
+ uint32_t otx_read_index;
+ /** This index aids in finding the window in the queue where OCTEON 9
+ * has read the commands.
+ */
+ uint32_t flush_index;
+ /* This keeps track of the instructions pending in this queue. */
+ uint64_t instr_pending;
+
+ /* Memory zone */
+ const struct rte_memzone *iq_mz;
+ /* OTX_EP doorbell register for the ring. */
+ void *doorbell_reg;
+
+ /* OTX_EP instruction count register for this ring. */
+ void *inst_cnt_reg;
+
+ /* Number of instructions pending to be posted to OCTEON 9. */
+ uint32_t fill_cnt;
+
struct otx_ep_device *otx_ep_dev;
uint32_t q_no;
@@ -219,54 +264,21 @@ struct otx_ep_instr_queue {
/* Size of the descriptor. */
uint8_t desc_size;
- /* Input ring index, where the driver should write the next packet */
- uint32_t host_write_index;
-
- /* Input ring index, where the OCTEON 9 should read the next packet */
- uint32_t otx_read_index;
-
uint32_t reset_instr_cnt;
- /** This index aids in finding the window in the queue where OCTEON 9
- * has read the commands.
- */
- uint32_t flush_index;
-
/* Free-running/wrapping instruction counter for IQ. */
uint32_t inst_cnt;
- /* This keeps track of the instructions pending in this queue. */
- uint64_t instr_pending;
-
- /* Pointer to the Virtual Base addr of the input ring. */
- uint8_t *base_addr;
+ uint64_t partial_ih;
/* This IQ request list */
struct otx_ep_instr_list *req_list;
- /* OTX_EP doorbell register for the ring. */
- void *doorbell_reg;
-
- /* OTX_EP instruction count register for this ring. */
- void *inst_cnt_reg;
-
- /* Number of instructions pending to be posted to OCTEON 9. */
- uint32_t fill_cnt;
-
/* Statistics for this input queue. */
struct otx_ep_iq_stats stats;
/* DMA mapped base address of the input descriptor ring. */
uint64_t base_addr_dma;
-
- /* Memory zone */
- const struct rte_memzone *iq_mz;
-
- /* Location in memory updated by SDP ISM */
- uint32_t *inst_cnt_ism;
-
- /* track inst count locally to consolidate HW counter updates */
- uint32_t inst_cnt_ism_prev;
};
/** Descriptor format.
@@ -344,14 +356,17 @@ struct otx_ep_oq_config {
/* The Descriptor Ring Output Queue(DROQ) structure. */
struct otx_ep_droq {
- struct otx_ep_device *otx_ep_dev;
/* The 8B aligned descriptor ring starts at this address. */
struct otx_ep_droq_desc *desc_ring;
- uint32_t q_no;
- uint64_t last_pkt_count;
+ /* The 8B aligned info ptrs begin from this address. */
+ struct otx_ep_droq_info *info_list;
- struct rte_mempool *mpool;
+ /* receive buffer list contains mbuf ptr list */
+ struct rte_mbuf **recv_buf_list;
+
+ /* Packets pending to be processed */
+ uint64_t pkts_pending;
/* Driver should read the next packet at this index */
uint32_t read_idx;
@@ -362,22 +377,17 @@ struct otx_ep_droq {
/* At this index, the driver will refill the descriptor's buffer */
uint32_t refill_idx;
- /* Packets pending to be processed */
- uint64_t pkts_pending;
+ /* The number of descriptors pending to refill. */
+ uint32_t refill_count;
/* Number of descriptors in this ring. */
uint32_t nb_desc;
- /* The number of descriptors pending to refill. */
- uint32_t refill_count;
-
uint32_t refill_threshold;
- /* The 8B aligned info ptrs begin from this address. */
- struct otx_ep_droq_info *info_list;
+ uint64_t last_pkt_count;
- /* receive buffer list contains mbuf ptr list */
- struct rte_mbuf **recv_buf_list;
+ struct rte_mempool *mpool;
/* The size of each buffer pointed by the buffer pointer. */
uint32_t buffer_size;
@@ -392,6 +402,13 @@ struct otx_ep_droq {
*/
void *pkts_sent_reg;
+ /* Pointer to host memory copy of output packet count, set by ISM */
+ uint32_t *pkts_sent_ism;
+ uint32_t pkts_sent_ism_prev;
+
+ /* Statistics for this DROQ. */
+ struct otx_ep_droq_stats stats;
+
/** Handle DMA incompletion during pkt reads.
* This variable is used to initiate a sent_reg_read
* that completes pending dma
@@ -400,8 +417,9 @@ struct otx_ep_droq {
*/
uint32_t sent_reg_val;
- /* Statistics for this DROQ. */
- struct otx_ep_droq_stats stats;
+ uint32_t q_no;
+
+ struct otx_ep_device *otx_ep_dev;
/* DMA mapped address of the DROQ descriptor ring. */
size_t desc_ring_dma;
@@ -419,10 +437,6 @@ struct otx_ep_droq {
const struct rte_memzone *desc_ring_mz;
const struct rte_memzone *info_mz;
-
- /* Pointer to host memory copy of output packet count, set by ISM */
- uint32_t *pkts_sent_ism;
- uint32_t pkts_sent_ism_prev;
};
#define OTX_EP_DROQ_SIZE (sizeof(struct otx_ep_droq))
@@ -545,6 +559,9 @@ struct otx_ep_device {
/* Negotiated Mbox version */
uint32_t mbox_neg_ver;
+
+ /* Generation */
+ uint32_t chip_gen;
};
int otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no,
diff --git a/drivers/net/octeon_ep/otx_ep_ethdev.c
b/drivers/net/octeon_ep/otx_ep_ethdev.c
index 970372bbd7..615cbbb648 100644
--- a/drivers/net/octeon_ep/otx_ep_ethdev.c
+++ b/drivers/net/octeon_ep/otx_ep_ethdev.c
@@ -27,6 +27,46 @@ static const struct rte_eth_desc_lim otx_ep_tx_desc_lim = {
.nb_align = OTX_EP_TXD_ALIGN,
};
+static void
+otx_ep_set_tx_func(struct rte_eth_dev *eth_dev)
+{
+ struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev);
+
+ if (otx_epvf->chip_gen == OTX_EP_CN10XX || otx_epvf->chip_gen ==
OTX_EP_CN9XX) {
+ eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts;
+ if (otx_epvf->tx_offloads & RTE_ETH_TX_OFFLOAD_MULTI_SEGS)
+ eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts_mseg;
+ } else {
+ eth_dev->tx_pkt_burst = &otx_ep_xmit_pkts;
+ }
+
+ if (eth_dev->data->dev_started)
+ rte_eth_fp_ops[eth_dev->data->port_id].tx_pkt_burst =
+ eth_dev->tx_pkt_burst;
+}
+
+static void
+otx_ep_set_rx_func(struct rte_eth_dev *eth_dev)
+{
+ struct otx_ep_device *otx_epvf = OTX_EP_DEV(eth_dev);
+
+ if (otx_epvf->chip_gen == OTX_EP_CN10XX) {
+ eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts;
+ if (otx_epvf->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+ eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts_mseg;
+ } else if (otx_epvf->chip_gen == OTX_EP_CN9XX) {
+ eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts;
+ if (otx_epvf->rx_offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
+ eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts_mseg;
+ } else {
+ eth_dev->rx_pkt_burst = &otx_ep_recv_pkts;
+ }
+
+ if (eth_dev->data->dev_started)
+ rte_eth_fp_ops[eth_dev->data->port_id].rx_pkt_burst =
+ eth_dev->rx_pkt_burst;
+}
+
static int
otx_ep_dev_info_get(struct rte_eth_dev *eth_dev,
struct rte_eth_dev_info *devinfo)
@@ -154,6 +194,10 @@ otx_ep_dev_start(struct rte_eth_dev *eth_dev)
}
otx_ep_dev_link_update(eth_dev, 0);
+
+ otx_ep_set_tx_func(eth_dev);
+ otx_ep_set_rx_func(eth_dev);
+
otx_ep_info("dev started\n");
for (q = 0; q < eth_dev->data->nb_rx_queues; q++)
@@ -266,18 +310,23 @@ otx_epdev_init(struct otx_ep_device *otx_epvf)
otx_epvf->fn_list.setup_device_regs(otx_epvf);
+ otx_epvf->eth_dev->tx_pkt_burst = &cnxk_ep_xmit_pkts;
otx_epvf->eth_dev->rx_pkt_burst = &otx_ep_recv_pkts;
- if (otx_epvf->chip_id == PCI_DEVID_OCTEONTX_EP_VF)
+ if (otx_epvf->chip_id == PCI_DEVID_OCTEONTX_EP_VF) {
otx_epvf->eth_dev->tx_pkt_burst = &otx_ep_xmit_pkts;
- else if (otx_epvf->chip_id == PCI_DEVID_CN9K_EP_NET_VF ||
+ otx_epvf->chip_gen = OTX_EP_CN8XX;
+ } else if (otx_epvf->chip_id == PCI_DEVID_CN9K_EP_NET_VF ||
otx_epvf->chip_id == PCI_DEVID_CN98XX_EP_NET_VF ||
otx_epvf->chip_id == PCI_DEVID_CNF95N_EP_NET_VF ||
- otx_epvf->chip_id == PCI_DEVID_CNF95O_EP_NET_VF ||
- otx_epvf->chip_id == PCI_DEVID_CN10KA_EP_NET_VF ||
- otx_epvf->chip_id == PCI_DEVID_CN10KB_EP_NET_VF ||
- otx_epvf->chip_id == PCI_DEVID_CNF10KA_EP_NET_VF ||
- otx_epvf->chip_id == PCI_DEVID_CNF10KB_EP_NET_VF) {
- otx_epvf->eth_dev->tx_pkt_burst = &otx2_ep_xmit_pkts;
+ otx_epvf->chip_id == PCI_DEVID_CNF95O_EP_NET_VF) {
+ otx_epvf->eth_dev->rx_pkt_burst = &cn9k_ep_recv_pkts;
+ otx_epvf->chip_gen = OTX_EP_CN9XX;
+ } else if (otx_epvf->chip_id == PCI_DEVID_CN10KA_EP_NET_VF ||
+ otx_epvf->chip_id == PCI_DEVID_CN10KB_EP_NET_VF ||
+ otx_epvf->chip_id == PCI_DEVID_CNF10KA_EP_NET_VF ||
+ otx_epvf->chip_id == PCI_DEVID_CNF10KB_EP_NET_VF) {
+ otx_epvf->eth_dev->rx_pkt_burst = &cnxk_ep_recv_pkts;
+ otx_epvf->chip_gen = OTX_EP_CN10XX;
} else {
otx_ep_err("Invalid chip_id\n");
ret = -EINVAL;
@@ -667,8 +716,8 @@ otx_ep_eth_dev_init(struct rte_eth_dev *eth_dev)
/* Single process support */
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
eth_dev->dev_ops = &otx_ep_eth_dev_ops;
- eth_dev->rx_pkt_burst = &otx_ep_recv_pkts;
- eth_dev->tx_pkt_burst = &otx2_ep_xmit_pkts;
+ otx_ep_set_tx_func(eth_dev);
+ otx_ep_set_rx_func(eth_dev);
return 0;
}
diff --git a/drivers/net/octeon_ep/otx_ep_rxtx.c
b/drivers/net/octeon_ep/otx_ep_rxtx.c
index 4c509a419f..c421ef0a1c 100644
--- a/drivers/net/octeon_ep/otx_ep_rxtx.c
+++ b/drivers/net/octeon_ep/otx_ep_rxtx.c
@@ -13,15 +13,8 @@
#include "otx_ep_common.h"
#include "otx_ep_vf.h"
-#include "otx2_ep_vf.h"
#include "otx_ep_rxtx.h"
-/* SDP_LENGTH_S specifies packet length and is of 8-byte size */
-#define OTX_EP_INFO_SIZE 8
-#define OTX_EP_FSZ_FS0 0
-#define DROQ_REFILL_THRESHOLD 16
-#define OTX2_SDP_REQUEST_ISM (0x1ULL << 63)
-
static void
otx_ep_dmazone_free(const struct rte_memzone *mz)
{
@@ -144,6 +137,13 @@ otx_ep_init_instr_queue(struct otx_ep_device *otx_ep, int
iq_no, int num_descs,
iq_no, iq->base_addr, (unsigned long)iq->base_addr_dma,
iq->nb_desc);
+ iq->mbuf_list = rte_zmalloc_socket("mbuf_list", (iq->nb_desc *
sizeof(struct rte_mbuf *)),
+ RTE_CACHE_LINE_SIZE,
rte_socket_id());
+ if (!iq->mbuf_list) {
+ otx_ep_err("IQ[%d] mbuf_list alloc failed\n", iq_no);
+ goto iq_init_fail;
+ }
+
iq->otx_ep_dev = otx_ep;
iq->q_no = iq_no;
iq->fill_cnt = 0;
@@ -675,85 +675,6 @@ otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts,
uint16_t nb_pkts)
return count;
}
-/* Enqueue requests/packets to OTX_EP IQ queue.
- * returns number of requests enqueued successfully
- */
-uint16_t
-otx2_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts)
-{
- struct otx_ep_instr_queue *iq = (struct otx_ep_instr_queue *)tx_queue;
- struct otx_ep_device *otx_ep = iq->otx_ep_dev;
- struct otx2_ep_instr_64B iqcmd2;
- uint32_t iqreq_type;
- struct rte_mbuf *m;
- uint32_t pkt_len;
- int count = 0;
- uint16_t i;
- int dbell;
- int index;
-
- iqcmd2.ih.u64 = 0;
- iqcmd2.irh.u64 = 0;
-
- /* ih invars */
- iqcmd2.ih.s.fsz = OTX_EP_FSZ_FS0;
- iqcmd2.ih.s.pkind = otx_ep->pkind; /* The SDK decided PKIND value */
- /* irh invars */
- iqcmd2.irh.s.opcode = OTX_EP_NW_PKT_OP;
-
- for (i = 0; i < nb_pkts; i++) {
- m = pkts[i];
- if (m->nb_segs == 1) {
- pkt_len = rte_pktmbuf_data_len(m);
- iqcmd2.ih.s.tlen = pkt_len + iqcmd2.ih.s.fsz;
- iqcmd2.dptr = rte_mbuf_data_iova(m); /*dptr*/
- iqcmd2.ih.s.gather = 0;
- iqcmd2.ih.s.gsz = 0;
- iqreq_type = OTX_EP_REQTYPE_NORESP_NET;
- } else {
- if (!(otx_ep->tx_offloads &
RTE_ETH_TX_OFFLOAD_MULTI_SEGS))
- goto xmit_fail;
-
- if (unlikely(prepare_xmit_gather_list(iq, m,
&iqcmd2.dptr, &iqcmd2.ih) < 0))
- goto xmit_fail;
-
- pkt_len = rte_pktmbuf_pkt_len(m);
- iqreq_type = OTX_EP_REQTYPE_NORESP_GATHER;
- }
-
- iqcmd2.irh.u64 = rte_bswap64(iqcmd2.irh.u64);
-
-#ifdef OTX_EP_IO_DEBUG
- otx_ep_dbg("After swapping\n");
- otx_ep_dbg("Word0 [dptr]: 0x%016lx\n",
- (unsigned long)iqcmd.dptr);
- otx_ep_dbg("Word1 [ihtx]: 0x%016lx\n", (unsigned long)iqcmd.ih);
- otx_ep_dbg("Word2 [pki_ih3]: 0x%016lx\n",
- (unsigned long)iqcmd.pki_ih3);
- otx_ep_dbg("Word3 [rptr]: 0x%016lx\n",
- (unsigned long)iqcmd.rptr);
- otx_ep_dbg("Word4 [irh]: 0x%016lx\n", (unsigned long)iqcmd.irh);
- otx_ep_dbg("Word5 [exhdr[0]]: 0x%016lx\n",
- (unsigned long)iqcmd.exhdr[0]);
-#endif
- index = iq->host_write_index;
- dbell = (i == (unsigned int)(nb_pkts - 1)) ? 1 : 0;
- if (otx_ep_send_data(otx_ep, iq, &iqcmd2, dbell))
- goto xmit_fail;
- otx_ep_iqreq_add(iq, m, iqreq_type, index);
- iq->stats.tx_pkts++;
- iq->stats.tx_bytes += pkt_len;
- count++;
- }
-
-xmit_fail:
- if (iq->instr_pending >= OTX_EP_MAX_INSTR)
- otx_ep_flush_iq(iq);
-
- /* Return no# of instructions posted successfully. */
- return count;
-}
-
static uint32_t
otx_ep_droq_refill(struct otx_ep_droq *droq)
{
diff --git a/drivers/net/octeon_ep/otx_ep_rxtx.h
b/drivers/net/octeon_ep/otx_ep_rxtx.h
index 3f12527004..cb68ef3b41 100644
--- a/drivers/net/octeon_ep/otx_ep_rxtx.h
+++ b/drivers/net/octeon_ep/otx_ep_rxtx.h
@@ -7,29 +7,53 @@
#include <rte_byteorder.h>
-#define OTX_EP_RXD_ALIGN 2
-#define OTX_EP_TXD_ALIGN 2
+#define OTX_EP_RXD_ALIGN 8
+#define OTX_EP_TXD_ALIGN 8
#define OTX_EP_IQ_SEND_FAILED (-1)
#define OTX_EP_IQ_SEND_SUCCESS (0)
-#define OTX_EP_MAX_DELAYED_PKT_RETRIES 10000
+#define OTX_EP_MAX_DELAYED_PKT_RETRIES 10
#define OTX_EP_FSZ 28
#define OTX2_EP_FSZ 24
-#define OTX_EP_MAX_INSTR 16
+#define OTX_EP_MAX_INSTR 128
+
+/* SDP_LENGTH_S specifies packet length and is of 8-byte size */
+#define OTX_EP_INFO_SIZE 8
+#define DROQ_REFILL_THRESHOLD 16
+#define OTX2_SDP_REQUEST_ISM (0x1ULL << 63)
static inline uint32_t
otx_ep_incr_index(uint32_t index, uint32_t count, uint32_t max)
{
return ((index + count) & (max - 1));
}
+
uint16_t
otx_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts);
+
uint16_t
otx2_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts);
+
+uint16_t
+otx_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t budget);
+
+uint16_t
+cnxk_ep_xmit_pkts(void *tx_queue, struct rte_mbuf **pkts, uint16_t nb_pkts);
+
+uint16_t
+cnxk_ep_xmit_pkts_mseg(void *tx_queue, struct rte_mbuf **pkts, uint16_t
nb_pkts);
+
+uint16_t
+cnxk_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t budget);
+
+uint16_t
+cnxk_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t
budget);
+
+uint16_t
+cn9k_ep_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t budget);
+
uint16_t
-otx_ep_recv_pkts(void *rx_queue,
- struct rte_mbuf **rx_pkts,
- uint16_t budget);
+cn9k_ep_recv_pkts_mseg(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t
budget);
#endif /* _OTX_EP_RXTX_H_ */
--
2.25.1
