From: Pavan Nikhilesh <pbhagavat...@marvell.com> Add event eth Tx adapter support to octeontx2 SSO.
Signed-off-by: Jerin Jacob <jer...@marvell.com> Signed-off-by: Pavan Nikhilesh <pbhagavat...@marvell.com> Signed-off-by: Nithin Dabilpuram <ndabilpu...@marvell.com> --- drivers/event/octeontx2/otx2_evdev.c | 78 ++++++++++++++++++++++ drivers/event/octeontx2/otx2_evdev.h | 26 ++++++++ drivers/event/octeontx2/otx2_evdev_adptr.c | 32 +++++++++ drivers/event/octeontx2/otx2_worker.c | 29 ++++++++ drivers/event/octeontx2/otx2_worker.h | 68 +++++++++++++++++++ drivers/event/octeontx2/otx2_worker_dual.c | 35 ++++++++++ 6 files changed, 268 insertions(+) diff --git a/drivers/event/octeontx2/otx2_evdev.c b/drivers/event/octeontx2/otx2_evdev.c index 138f4e4fa..4fdc02bf1 100644 --- a/drivers/event/octeontx2/otx2_evdev.c +++ b/drivers/event/octeontx2/otx2_evdev.c @@ -168,6 +168,39 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R }; + /* Tx modes */ + const event_tx_adapter_enqueue ssogws_tx_adptr_enq[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = otx2_ssogws_tx_adptr_enq_ ## name, +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + }; + + const event_tx_adapter_enqueue + ssogws_tx_adptr_enq_seg[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = otx2_ssogws_tx_adptr_enq_seg_ ## name, +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + }; + + const event_tx_adapter_enqueue + ssogws_dual_tx_adptr_enq[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = otx2_ssogws_dual_tx_adptr_enq_ ## name, +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + }; + + const event_tx_adapter_enqueue + ssogws_dual_tx_adptr_enq_seg[2][2][2][2][2] = { +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ + [f4][f3][f2][f1][f0] = \ + otx2_ssogws_dual_tx_adptr_enq_seg_ ## name, +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + }; + event_dev->enqueue = otx2_ssogws_enq; event_dev->enqueue_burst = otx2_ssogws_enq_burst; event_dev->enqueue_new_burst = otx2_ssogws_enq_new_burst; @@ -238,6 +271,23 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC } } + if (dev->tx_offloads & NIX_TX_MULTI_SEG_F) { + /* [TSMP] [MBUF_NOFF] [VLAN] [OL3_L4_CSUM] [L3_L4_CSUM] */ + event_dev->txa_enqueue = ssogws_tx_adptr_enq_seg + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)]; + } else { + event_dev->txa_enqueue = ssogws_tx_adptr_enq + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_MBUF_NOFF_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_VLAN_QINQ_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_L3_L4_CSUM_F)]; + } + if (dev->dual_ws) { event_dev->enqueue = otx2_ssogws_dual_enq; event_dev->enqueue_burst = otx2_ssogws_dual_enq_burst; @@ -352,6 +402,31 @@ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC NIX_RX_OFFLOAD_RSS_F)]; } } + + if (dev->tx_offloads & NIX_TX_MULTI_SEG_F) { + /* [TSMP] [MBUF_NOFF] [VLAN] [OL3_L4_CSUM] [L3_L4_CSUM] */ + event_dev->txa_enqueue = ssogws_dual_tx_adptr_enq_seg + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_MBUF_NOFF_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_VLAN_QINQ_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_L3_L4_CSUM_F)]; + } else { + event_dev->txa_enqueue = ssogws_dual_tx_adptr_enq + [!!(dev->tx_offloads & NIX_TX_OFFLOAD_TSTAMP_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_MBUF_NOFF_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_VLAN_QINQ_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_OL3_OL4_CSUM_F)] + [!!(dev->tx_offloads & + NIX_TX_OFFLOAD_L3_L4_CSUM_F)]; + } } rte_mb(); } @@ -1410,6 +1485,9 @@ static struct rte_eventdev_ops otx2_sso_ops = { .eth_rx_adapter_start = otx2_sso_rx_adapter_start, .eth_rx_adapter_stop = otx2_sso_rx_adapter_stop, + .eth_tx_adapter_caps_get = otx2_sso_tx_adapter_caps_get, + .eth_tx_adapter_queue_add = otx2_sso_tx_adapter_queue_add, + .timer_adapter_caps_get = otx2_tim_caps_get, .xstats_get = otx2_sso_xstats_get, diff --git a/drivers/event/octeontx2/otx2_evdev.h b/drivers/event/octeontx2/otx2_evdev.h index dd493715b..0f0be341f 100644 --- a/drivers/event/octeontx2/otx2_evdev.h +++ b/drivers/event/octeontx2/otx2_evdev.h @@ -8,6 +8,7 @@ #include <rte_eventdev.h> #include <rte_eventdev_pmd.h> #include <rte_event_eth_rx_adapter.h> +#include <rte_event_eth_tx_adapter.h> #include "otx2_common.h" #include "otx2_dev.h" @@ -118,6 +119,7 @@ struct otx2_sso_evdev { rte_iova_t fc_iova; struct rte_mempool *xaq_pool; uint64_t rx_offloads; + uint64_t tx_offloads; uint16_t rx_adptr_pool_cnt; uint32_t adptr_xae_cnt; uint64_t *rx_adptr_pools; @@ -308,6 +310,22 @@ uint16_t otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port, \ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +uint16_t otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[],\ + uint16_t nb_events); \ +uint16_t otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, \ + struct rte_event ev[], \ + uint16_t nb_events); \ +uint16_t otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port, \ + struct rte_event ev[], \ + uint16_t nb_events); \ +uint16_t otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port, \ + struct rte_event ev[], \ + uint16_t nb_events); \ + +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + void sso_updt_xae_cnt(struct otx2_sso_evdev *dev, void *data, uint32_t event_type); int sso_xae_reconfigure(struct rte_eventdev *event_dev); @@ -327,6 +345,14 @@ int otx2_sso_rx_adapter_start(const struct rte_eventdev *event_dev, const struct rte_eth_dev *eth_dev); int otx2_sso_rx_adapter_stop(const struct rte_eventdev *event_dev, const struct rte_eth_dev *eth_dev); +int otx2_sso_tx_adapter_caps_get(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, + uint32_t *caps); +int otx2_sso_tx_adapter_queue_add(uint8_t id, + const struct rte_eventdev *event_dev, + const struct rte_eth_dev *eth_dev, + int32_t tx_queue_id); + /* Clean up API's */ typedef void (*otx2_handle_event_t)(void *arg, struct rte_event ev); void ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id, diff --git a/drivers/event/octeontx2/otx2_evdev_adptr.c b/drivers/event/octeontx2/otx2_evdev_adptr.c index c2cf4196e..62ae5b61c 100644 --- a/drivers/event/octeontx2/otx2_evdev_adptr.c +++ b/drivers/event/octeontx2/otx2_evdev_adptr.c @@ -327,3 +327,35 @@ otx2_sso_rx_adapter_stop(const struct rte_eventdev *event_dev, return 0; } + +int +otx2_sso_tx_adapter_caps_get(const struct rte_eventdev *dev, + const struct rte_eth_dev *eth_dev, uint32_t *caps) +{ + int ret; + + RTE_SET_USED(dev); + ret = strncmp(eth_dev->device->driver->name, "net_octeontx2,", 13); + if (ret) + *caps = 0; + else + *caps = RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT; + + return 0; +} + +int +otx2_sso_tx_adapter_queue_add(uint8_t id, const struct rte_eventdev *event_dev, + const struct rte_eth_dev *eth_dev, + int32_t tx_queue_id) +{ + struct otx2_eth_dev *otx2_eth_dev = eth_dev->data->dev_private; + struct otx2_sso_evdev *dev = sso_pmd_priv(event_dev); + + RTE_SET_USED(id); + RTE_SET_USED(tx_queue_id); + dev->tx_offloads |= otx2_eth_dev->tx_offload_flags; + sso_fastpath_fns_set((struct rte_eventdev *)(uintptr_t)event_dev); + + return 0; +} diff --git a/drivers/event/octeontx2/otx2_worker.c b/drivers/event/octeontx2/otx2_worker.c index ea2d0b5a4..cd14cd3d2 100644 --- a/drivers/event/octeontx2/otx2_worker.c +++ b/drivers/event/octeontx2/otx2_worker.c @@ -267,6 +267,35 @@ otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[], return 1; } +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +uint16_t __hot \ +otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ +{ \ + struct otx2_ssogws *ws = port; \ + uint64_t cmd[sz]; \ + \ + RTE_SET_USED(nb_events); \ + return otx2_ssogws_event_tx(ws, ev, cmd, flags); \ +} +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +uint16_t __hot \ +otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port, struct rte_event ev[],\ + uint16_t nb_events) \ +{ \ + struct otx2_ssogws *ws = port; \ + uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \ + \ + RTE_SET_USED(nb_events); \ + return otx2_ssogws_event_tx(ws, ev, cmd, (flags) | \ + NIX_TX_MULTI_SEG_F); \ +} +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + void ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id, uintptr_t base, otx2_handle_event_t fn, void *arg) diff --git a/drivers/event/octeontx2/otx2_worker.h b/drivers/event/octeontx2/otx2_worker.h index 9d116f1ac..9c13cb1f0 100644 --- a/drivers/event/octeontx2/otx2_worker.h +++ b/drivers/event/octeontx2/otx2_worker.h @@ -218,4 +218,72 @@ otx2_ssogws_swtag_wait(struct otx2_ssogws *ws) #endif } +static __rte_always_inline void +otx2_ssogws_head_wait(struct otx2_ssogws *ws) +{ + while (!(otx2_read64(ws->tag_op) & BIT_ULL(35))) + ; +} + +static __rte_always_inline const struct otx2_eth_txq * +otx2_ssogws_xtract_meta(struct rte_mbuf *m) +{ + return rte_eth_devices[m->port].data->tx_queues[ + rte_event_eth_tx_adapter_txq_get(m)]; +} + +static __rte_always_inline void +otx2_ssogws_prepare_pkt(const struct otx2_eth_txq *txq, struct rte_mbuf *m, + uint64_t *cmd, const uint32_t flags) +{ + otx2_lmt_mov(cmd, txq->cmd, otx2_nix_tx_ext_subs(flags)); + otx2_nix_xmit_prepare(m, cmd, flags); +} + +static __rte_always_inline uint16_t +otx2_ssogws_event_tx(struct otx2_ssogws *ws, struct rte_event ev[], + uint64_t *cmd, const uint32_t flags) +{ + const struct otx2_eth_txq *txq = NULL; + struct rte_mbuf *m = ev[0].mbuf; + uint16_t segdw; + + rte_prefetch_non_temporal(m); + + switch (ev->sched_type) { + case SSO_SYNC_ORDERED: + rte_cio_wmb(); + txq = otx2_ssogws_xtract_meta(m); + otx2_ssogws_prepare_pkt(txq, m, cmd, flags); + otx2_ssogws_head_wait(ws); + break; + case SSO_SYNC_UNTAGGED: + otx2_ssogws_swtag_norm(ws, ev->event, SSO_SYNC_ATOMIC); + rte_cio_wmb(); + txq = otx2_ssogws_xtract_meta(m); + otx2_ssogws_prepare_pkt(txq, m, cmd, flags); + otx2_ssogws_swtag_wait(ws); + break; + case SSO_SYNC_ATOMIC: + rte_cio_wmb(); + txq = otx2_ssogws_xtract_meta(m); + otx2_ssogws_prepare_pkt(txq, m, cmd, flags); + break; + } + + if (flags & NIX_TX_MULTI_SEG_F) { + segdw = otx2_nix_prepare_mseg(m, cmd, flags); + otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0], + m->ol_flags, segdw, flags); + otx2_nix_xmit_mseg_one(cmd, txq->lmt_addr, txq->io_addr, segdw); + } else { + /* Passing no of segdw as 4: HDR + EXT + SG + SMEM */ + otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0], + m->ol_flags, 4, flags); + otx2_nix_xmit_one(cmd, txq->lmt_addr, txq->io_addr, flags); + } + + return 1; +} + #endif diff --git a/drivers/event/octeontx2/otx2_worker_dual.c b/drivers/event/octeontx2/otx2_worker_dual.c index cbe03c1bb..37c274a54 100644 --- a/drivers/event/octeontx2/otx2_worker_dual.c +++ b/drivers/event/octeontx2/otx2_worker_dual.c @@ -305,3 +305,38 @@ otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port, \ SSO_RX_ADPTR_ENQ_FASTPATH_FUNC #undef R + +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +uint16_t __hot \ +otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port, \ + struct rte_event ev[], \ + uint16_t nb_events) \ +{ \ + struct otx2_ssogws_dual *ws = port; \ + struct otx2_ssogws *vws = \ + (struct otx2_ssogws *)&ws->ws_state[!ws->vws]; \ + uint64_t cmd[sz]; \ + \ + RTE_SET_USED(nb_events); \ + return otx2_ssogws_event_tx(vws, ev, cmd, flags); \ +} +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + +#define T(name, f4, f3, f2, f1, f0, sz, flags) \ +uint16_t __hot \ +otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port, \ + struct rte_event ev[], \ + uint16_t nb_events) \ +{ \ + struct otx2_ssogws_dual *ws = port; \ + struct otx2_ssogws *vws = \ + (struct otx2_ssogws *)&ws->ws_state[!ws->vws]; \ + uint64_t cmd[(sz) + NIX_TX_MSEG_SG_DWORDS - 2]; \ + \ + RTE_SET_USED(nb_events); \ + return otx2_ssogws_event_tx(vws, ev, cmd, (flags) | \ + NIX_TX_MULTI_SEG_F); \ +} +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T -- 2.21.0