Adding macro based framework to hook dequeue/enqueue function pointers to the appropriate function based on rx/tx offloads.
Signed-off-by: Harman Kalra <hka...@marvell.com> --- drivers/event/octeontx/ssovf_evdev.c | 36 ---- drivers/event/octeontx/ssovf_evdev.h | 24 +-- drivers/event/octeontx/ssovf_worker.c | 259 ++++++++++++++------------ drivers/net/octeontx/octeontx_rxtx.h | 7 + 4 files changed, 150 insertions(+), 176 deletions(-) diff --git a/drivers/event/octeontx/ssovf_evdev.c b/drivers/event/octeontx/ssovf_evdev.c index 1024b7284..5d074bcbc 100644 --- a/drivers/event/octeontx/ssovf_evdev.c +++ b/drivers/event/octeontx/ssovf_evdev.c @@ -137,42 +137,6 @@ ssovf_mbox_timeout_ticks(uint64_t ns, uint64_t *tmo_ticks) return 0; } -static void -ssovf_fastpath_fns_set(struct rte_eventdev *dev) -{ - struct ssovf_evdev *edev = ssovf_pmd_priv(dev); - - dev->enqueue = ssows_enq; - dev->enqueue_burst = ssows_enq_burst; - dev->enqueue_new_burst = ssows_enq_new_burst; - dev->enqueue_forward_burst = ssows_enq_fwd_burst; - - if (!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)) { - dev->dequeue = ssows_deq_mseg; - dev->dequeue_burst = ssows_deq_burst_mseg; - - if (edev->is_timeout_deq) { - dev->dequeue = ssows_deq_timeout_mseg; - dev->dequeue_burst = ssows_deq_timeout_burst_mseg; - } - } else { - dev->dequeue = ssows_deq; - dev->dequeue_burst = ssows_deq_burst; - - if (edev->is_timeout_deq) { - dev->dequeue = ssows_deq_timeout; - dev->dequeue_burst = ssows_deq_timeout_burst; - } - } - - if (!!(edev->tx_offload_flags & OCCTX_TX_MULTI_SEG_F)) - dev->txa_enqueue = sso_event_tx_adapter_enqueue_mseg; - else - dev->txa_enqueue = sso_event_tx_adapter_enqueue; - - dev->txa_enqueue_same_dest = dev->txa_enqueue; -} - static void ssovf_info_get(struct rte_eventdev *dev, struct rte_event_dev_info *dev_info) { diff --git a/drivers/event/octeontx/ssovf_evdev.h b/drivers/event/octeontx/ssovf_evdev.h index 1c3ae8556..1f5066c9a 100644 --- a/drivers/event/octeontx/ssovf_evdev.h +++ b/drivers/event/octeontx/ssovf_evdev.h @@ -14,6 +14,9 @@ #include "octeontx_rxtx.h" +#define SSO_RX_ADPTR_ENQ_FASTPATH_FUNC OCCTX_RX_FASTPATH_MODES +#define SSO_TX_ADPTR_ENQ_FASTPATH_FUNC OCCTX_TX_FASTPATH_MODES + #define EVENTDEV_NAME_OCTEONTX_PMD event_octeontx #define SSOVF_LOG(level, fmt, args...) \ @@ -171,32 +174,13 @@ uint16_t ssows_enq_new_burst(void *port, const struct rte_event ev[], uint16_t nb_events); uint16_t ssows_enq_fwd_burst(void *port, const struct rte_event ev[], uint16_t nb_events); -uint16_t ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks); -uint16_t ssows_deq_burst(void *port, struct rte_event ev[], - uint16_t nb_events, uint64_t timeout_ticks); -uint16_t ssows_deq_timeout(void *port, struct rte_event *ev, - uint64_t timeout_ticks); -uint16_t ssows_deq_timeout_burst(void *port, struct rte_event ev[], - uint16_t nb_events, uint64_t timeout_ticks); -uint16_t ssows_deq_mseg(void *port, struct rte_event *ev, - uint64_t timeout_ticks); -uint16_t ssows_deq_burst_mseg(void *port, struct rte_event ev[], - uint16_t nb_events, uint64_t timeout_ticks); -uint16_t ssows_deq_timeout_mseg(void *port, struct rte_event *ev, - uint64_t timeout_ticks); -uint16_t ssows_deq_timeout_burst_mseg(void *port, struct rte_event ev[], - uint16_t nb_events, uint64_t timeout_ticks); - typedef void (*ssows_handle_event_t)(void *arg, struct rte_event ev); void ssows_flush_events(struct ssows *ws, uint8_t queue_id, ssows_handle_event_t fn, void *arg); void ssows_reset(struct ssows *ws); -uint16_t sso_event_tx_adapter_enqueue(void *port, - struct rte_event ev[], uint16_t nb_events); -uint16_t sso_event_tx_adapter_enqueue_mseg(void *port, - struct rte_event ev[], uint16_t nb_events); int ssovf_info(struct ssovf_info *info); void *ssovf_bar(enum ssovf_type, uint8_t id, uint8_t bar); int test_eventdev_octeontx(void); +void ssovf_fastpath_fns_set(struct rte_eventdev *dev); #endif /* __SSOVF_EVDEV_H__ */ diff --git a/drivers/event/octeontx/ssovf_worker.c b/drivers/event/octeontx/ssovf_worker.c index a811c2252..b5873c3fa 100644 --- a/drivers/event/octeontx/ssovf_worker.c +++ b/drivers/event/octeontx/ssovf_worker.c @@ -91,112 +91,62 @@ ssows_release_event(struct ssows *ws) ssows_swtag_untag(ws); } -__rte_always_inline uint16_t __rte_hot -ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks) -{ - struct ssows *ws = port; - - RTE_SET_USED(timeout_ticks); - - if (ws->swtag_req) { - ws->swtag_req = 0; - ssows_swtag_wait(ws); - return 1; - } else { - return ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE); - } -} - -__rte_always_inline uint16_t __rte_hot -ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks) -{ - struct ssows *ws = port; - uint64_t iter; - uint16_t ret = 1; - - if (ws->swtag_req) { - ws->swtag_req = 0; - ssows_swtag_wait(ws); - } else { - ret = ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE); - for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) - ret = ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE); - } - return ret; +#define R(name, f0, flags) \ +static uint16_t __rte_noinline __rte_hot \ +ssows_deq_ ##name(void *port, struct rte_event *ev, uint64_t timeout_ticks) \ +{ \ + struct ssows *ws = port; \ + \ + RTE_SET_USED(timeout_ticks); \ + \ + if (ws->swtag_req) { \ + ws->swtag_req = 0; \ + ssows_swtag_wait(ws); \ + return 1; \ + } else { \ + return ssows_get_work(ws, ev, flags); \ + } \ +} \ + \ +static uint16_t __rte_hot \ +ssows_deq_burst_ ##name(void *port, struct rte_event ev[], \ + uint16_t nb_events, uint64_t timeout_ticks) \ +{ \ + RTE_SET_USED(nb_events); \ + \ + return ssows_deq_ ##name(port, ev, timeout_ticks); \ +} \ + \ +static uint16_t __rte_hot \ +ssows_deq_timeout_ ##name(void *port, struct rte_event *ev, \ + uint64_t timeout_ticks) \ +{ \ + struct ssows *ws = port; \ + uint64_t iter; \ + uint16_t ret = 1; \ + \ + if (ws->swtag_req) { \ + ws->swtag_req = 0; \ + ssows_swtag_wait(ws); \ + } else { \ + ret = ssows_get_work(ws, ev, flags); \ + for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \ + ret = ssows_get_work(ws, ev, flags); \ + } \ + return ret; \ +} \ + \ +static uint16_t __rte_hot \ +ssows_deq_timeout_burst_ ##name(void *port, struct rte_event ev[], \ + uint16_t nb_events, uint64_t timeout_ticks) \ +{ \ + RTE_SET_USED(nb_events); \ + \ + return ssows_deq_timeout_ ##name(port, ev, timeout_ticks); \ } -uint16_t __rte_hot -ssows_deq_burst(void *port, struct rte_event ev[], uint16_t nb_events, - uint64_t timeout_ticks) -{ - RTE_SET_USED(nb_events); - - return ssows_deq(port, ev, timeout_ticks); -} - -uint16_t __rte_hot -ssows_deq_timeout_burst(void *port, struct rte_event ev[], uint16_t nb_events, - uint64_t timeout_ticks) -{ - RTE_SET_USED(nb_events); - - return ssows_deq_timeout(port, ev, timeout_ticks); -} - -__rte_always_inline uint16_t __rte_hot -ssows_deq_mseg(void *port, struct rte_event *ev, uint64_t timeout_ticks) -{ - struct ssows *ws = port; - - RTE_SET_USED(timeout_ticks); - - if (ws->swtag_req) { - ws->swtag_req = 0; - ssows_swtag_wait(ws); - return 1; - } else { - return ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE | - OCCTX_RX_MULTI_SEG_F); - } -} - -__rte_always_inline uint16_t __rte_hot -ssows_deq_timeout_mseg(void *port, struct rte_event *ev, uint64_t timeout_ticks) -{ - struct ssows *ws = port; - uint64_t iter; - uint16_t ret = 1; - - if (ws->swtag_req) { - ws->swtag_req = 0; - ssows_swtag_wait(ws); - } else { - ret = ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE | - OCCTX_RX_MULTI_SEG_F); - for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) - ret = ssows_get_work(ws, ev, OCCTX_RX_OFFLOAD_NONE | - OCCTX_RX_MULTI_SEG_F); - } - return ret; -} - -uint16_t __rte_hot -ssows_deq_burst_mseg(void *port, struct rte_event ev[], uint16_t nb_events, - uint64_t timeout_ticks) -{ - RTE_SET_USED(nb_events); - - return ssows_deq_mseg(port, ev, timeout_ticks); -} - -uint16_t __rte_hot -ssows_deq_timeout_burst_mseg(void *port, struct rte_event ev[], - uint16_t nb_events, uint64_t timeout_ticks) -{ - RTE_SET_USED(nb_events); - - return ssows_deq_timeout_mseg(port, ev, timeout_ticks); -} +SSO_RX_ADPTR_ENQ_FASTPATH_FUNC +#undef R __rte_always_inline uint16_t __rte_hot ssows_enq(void *port, const struct rte_event *ev) @@ -321,7 +271,8 @@ ssows_reset(struct ssows *ws) static __rte_always_inline uint16_t __sso_event_tx_adapter_enqueue(void *port, struct rte_event ev[], - uint16_t nb_events, const uint16_t flag) + uint16_t nb_events, uint64_t *cmd, + const uint16_t flag) { uint16_t port_id; uint16_t queue_id; @@ -329,9 +280,7 @@ __sso_event_tx_adapter_enqueue(void *port, struct rte_event ev[], struct rte_eth_dev *ethdev; struct ssows *ws = port; struct octeontx_txq *txq; - uint64_t cmd[4]; - RTE_SET_USED(nb_events); switch (ev->sched_type) { case SSO_SYNC_ORDERED: ssows_swtag_norm(ws, ev->event, SSO_SYNC_ATOMIC); @@ -355,22 +304,92 @@ __sso_event_tx_adapter_enqueue(void *port, struct rte_event ev[], ethdev = &rte_eth_devices[port_id]; txq = ethdev->data->tx_queues[queue_id]; - return __octeontx_xmit_pkts(txq, &m, 1, cmd, flag); + return __octeontx_xmit_pkts(txq, &m, nb_events, cmd, flag); } -uint16_t -sso_event_tx_adapter_enqueue(void *port, struct rte_event ev[], - uint16_t nb_events) -{ - return __sso_event_tx_adapter_enqueue(port, ev, nb_events, - OCCTX_TX_OFFLOAD_NONE); +#define T(name, f3, f2, f1, f0, sz, flags) \ +static uint16_t __rte_noinline __rte_hot \ +sso_event_tx_adapter_enqueue_ ## name(void *port, struct rte_event ev[], \ + uint16_t nb_events) \ +{ \ + uint64_t cmd[sz]; \ + return __sso_event_tx_adapter_enqueue(port, ev, nb_events, cmd, \ + flags); \ } -uint16_t -sso_event_tx_adapter_enqueue_mseg(void *port, struct rte_event ev[], - uint16_t nb_events) +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + +void +ssovf_fastpath_fns_set(struct rte_eventdev *dev) { - return __sso_event_tx_adapter_enqueue(port, ev, nb_events, - OCCTX_TX_OFFLOAD_NONE | - OCCTX_TX_MULTI_SEG_F); + struct ssovf_evdev *edev = ssovf_pmd_priv(dev); + + dev->enqueue = ssows_enq; + dev->enqueue_burst = ssows_enq_burst; + dev->enqueue_new_burst = ssows_enq_new_burst; + dev->enqueue_forward_burst = ssows_enq_fwd_burst; + + const event_tx_adapter_enqueue ssow_txa_enqueue[2][2][2][2] = { +#define T(name, f3, f2, f1, f0, sz, flags) \ + [f3][f2][f1][f0] = sso_event_tx_adapter_enqueue_ ##name, + +SSO_TX_ADPTR_ENQ_FASTPATH_FUNC +#undef T + }; + + dev->txa_enqueue = ssow_txa_enqueue + [!!(edev->tx_offload_flags & OCCTX_TX_OFFLOAD_MBUF_NOFF_F)] + [0] + [0] + [!!(edev->tx_offload_flags & OCCTX_TX_MULTI_SEG_F)]; + + dev->txa_enqueue_same_dest = dev->txa_enqueue; + + /* Assigning dequeue func pointers */ + const event_dequeue_t ssow_deq[2] = { +#define R(name, f0, flags) \ + [f0] = ssows_deq_ ##name, + +SSO_RX_ADPTR_ENQ_FASTPATH_FUNC +#undef R + }; + + dev->dequeue = ssow_deq + [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; + + const event_dequeue_burst_t ssow_deq_burst[2] = { +#define R(name, f0, flags) \ + [f0] = ssows_deq_burst_ ##name, + +SSO_RX_ADPTR_ENQ_FASTPATH_FUNC +#undef R + }; + + dev->dequeue_burst = ssow_deq_burst + [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; + + if (edev->is_timeout_deq) { + const event_dequeue_t ssow_deq_timeout[2] = { +#define R(name, f0, flags) \ + [f0] = ssows_deq_timeout_ ##name, + +SSO_RX_ADPTR_ENQ_FASTPATH_FUNC +#undef R + }; + + dev->dequeue = ssow_deq_timeout + [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; + + const event_dequeue_burst_t ssow_deq_timeout_burst[2] = { +#define R(name, f0, flags) \ + [f0] = ssows_deq_timeout_burst_ ##name, + +SSO_RX_ADPTR_ENQ_FASTPATH_FUNC +#undef R + }; + + dev->dequeue_burst = ssow_deq_timeout_burst + [!!(edev->rx_offload_flags & OCCTX_RX_MULTI_SEG_F)]; + } } diff --git a/drivers/net/octeontx/octeontx_rxtx.h b/drivers/net/octeontx/octeontx_rxtx.h index 5e607f170..144ae055b 100644 --- a/drivers/net/octeontx/octeontx_rxtx.h +++ b/drivers/net/octeontx/octeontx_rxtx.h @@ -484,4 +484,11 @@ T(noff_ol3ol4csum_l3l4csum_mseg, 1, 1, 1, 1, 14, \ NOFF_F | OL3OL4CSUM_F | L3L4CSUM_F | \ MULT_F) +/* RX offload macros */ +#define MULT_RX_F OCCTX_RX_MULTI_SEG_F +/* [MULTI_SEG] */ +#define OCCTX_RX_FASTPATH_MODES \ +R(no_offload, 0, OCCTX_RX_OFFLOAD_NONE) \ +R(mseg, 1, MULT_RX_F) \ + #endif /* __OCTEONTX_RXTX_H__ */ -- 2.18.0