Added cnxk driver support for dma event enqueue and dequeue. Signed-off-by: Amit Prakash Shukla <amitpraka...@marvell.com> --- drivers/dma/cnxk/cnxk_dma_event_dp.h | 22 +++ drivers/dma/cnxk/cnxk_dmadev.h | 9 +- drivers/dma/cnxk/cnxk_dmadev_fp.c | 209 +++++++++++++++++++++++++++ drivers/dma/cnxk/meson.build | 6 +- drivers/dma/cnxk/version.map | 9 ++ 5 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 drivers/dma/cnxk/cnxk_dma_event_dp.h create mode 100644 drivers/dma/cnxk/version.map
diff --git a/drivers/dma/cnxk/cnxk_dma_event_dp.h b/drivers/dma/cnxk/cnxk_dma_event_dp.h new file mode 100644 index 0000000000..bf9b01f8f1 --- /dev/null +++ b/drivers/dma/cnxk/cnxk_dma_event_dp.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(C) 2023 Marvell. + */ + +#ifndef _CNXK_DMA_EVENT_DP_H_ +#define _CNXK_DMA_EVENT_DP_H_ + +#include <stdint.h> + +#include <rte_common.h> +#include <rte_eventdev.h> + +__rte_internal +uint16_t cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events); + +__rte_internal +uint16_t cn9k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events); + +__rte_internal +uintptr_t cnxk_dma_adapter_dequeue(uintptr_t get_work1); + +#endif /* _CNXK_DMA_EVENT_DP_H_ */ diff --git a/drivers/dma/cnxk/cnxk_dmadev.h b/drivers/dma/cnxk/cnxk_dmadev.h index 75059b8843..9cba388d02 100644 --- a/drivers/dma/cnxk/cnxk_dmadev.h +++ b/drivers/dma/cnxk/cnxk_dmadev.h @@ -40,6 +40,11 @@ */ #define CNXK_DPI_REQ_CDATA 0xFF +/* Set Completion data to 0xDEADBEEF when request submitted for SSO. + * This helps differentiate if the dequeue is called after cnxk enueue. + */ +#define CNXK_DPI_REQ_SSO_CDATA 0xDEADBEEF + union cnxk_dpi_instr_cmd { uint64_t u; struct cn9k_dpi_instr_cmd { @@ -85,7 +90,9 @@ union cnxk_dpi_instr_cmd { struct cnxk_dpi_compl_s { uint64_t cdata; - void *cb_data; + void *op; + uint16_t dev_id; + uint16_t vchan; uint32_t wqecs; }; diff --git a/drivers/dma/cnxk/cnxk_dmadev_fp.c b/drivers/dma/cnxk/cnxk_dmadev_fp.c index 16d7b5426b..c7cd036a5b 100644 --- a/drivers/dma/cnxk/cnxk_dmadev_fp.c +++ b/drivers/dma/cnxk/cnxk_dmadev_fp.c @@ -5,6 +5,8 @@ #include <rte_vect.h> #include "cnxk_dmadev.h" +#include "cnxk_dma_event_dp.h" +#include <rte_event_dma_adapter.h> static __plt_always_inline void __dpi_cpy_scalar(uint64_t *src, uint64_t *dst, uint8_t n) @@ -434,3 +436,210 @@ cn10k_dmadev_copy_sg(void *dev_private, uint16_t vchan, const struct rte_dma_sge return dpi_conf->desc_idx++; } + +uint16_t +cn10k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) +{ + union rte_event_dma_metadata *dma_mdata; + struct rte_event_dma_request *req_info; + const struct rte_dma_sge *src, *dst; + struct rte_event_dma_adapter_op *op; + struct cnxk_dpi_compl_s *comp_ptr; + struct cnxk_dpi_conf *dpi_conf; + struct cnxk_dpi_vf_s *dpivf; + struct rte_event *rsp_info; + uint16_t nb_src, nb_dst; + struct rte_dma_dev *dev; + uint64_t hdr[4]; + uint16_t count; + int rc; + + PLT_SET_USED(ws); + + for (count = 0; count < nb_events; count++) { + op = ev[count].event_ptr; + dma_mdata = (union rte_event_dma_metadata *)((uint8_t *)op + + sizeof(struct rte_event_dma_adapter_op)); + rsp_info = &dma_mdata->response_info; + req_info = &dma_mdata->request_info; + dev = rte_dma_pmd_dev_get(req_info->dma_dev_id); + dpivf = dev->data->dev_private; + dpi_conf = &dpivf->conf[req_info->vchan]; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & dpi_conf->c_desc.max_cnt) == + dpi_conf->c_desc.head)) + return count; + + comp_ptr = dpi_conf->c_desc.compl_ptr[dpi_conf->c_desc.tail]; + CNXK_DPI_STRM_INC(dpi_conf->c_desc, tail); + comp_ptr->op = op; + comp_ptr->dev_id = req_info->dma_dev_id; + comp_ptr->vchan = req_info->vchan; + comp_ptr->cdata = CNXK_DPI_REQ_SSO_CDATA; + + nb_src = op->nb_src & CNXK_DPI_MAX_POINTER; + nb_dst = op->nb_dst & CNXK_DPI_MAX_POINTER; + + hdr[0] = dpi_conf->cmd.u | ((uint64_t)DPI_HDR_PT_WQP << 54); + hdr[0] |= (nb_dst << 6) | nb_src; + hdr[1] = ((uint64_t)comp_ptr); + hdr[2] = (RTE_EVENT_TYPE_DMADEV << 28 | (rsp_info->sub_event_type << 20) | + rsp_info->flow_id); + hdr[2] |= ((uint64_t)(rsp_info->sched_type & DPI_HDR_TT_MASK)) << 32; + hdr[2] |= ((uint64_t)(rsp_info->queue_id & DPI_HDR_GRP_MASK)) << 34; + + src = &op->src_seg[0]; + dst = &op->dst_seg[0]; + + rc = __dpi_queue_write_sg(dpivf, hdr, src, dst, nb_src, nb_dst); + if (unlikely(rc)) { + CNXK_DPI_STRM_DEC(dpi_conf->c_desc, tail); + return rc; + } + + if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_wmb(); + plt_write64(dpi_conf->pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), + dpivf->rdpi.rbase + DPI_VDMA_DBELL); + dpi_conf->stats.submitted += dpi_conf->pending + 1; + dpi_conf->pnum_words = 0; + dpi_conf->pending = 0; + } else { + dpi_conf->pnum_words += CNXK_DPI_CMD_LEN(nb_src, nb_dst); + dpi_conf->pending++; + } + } + + return count; +} + +uint16_t +cn9k_dma_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events) +{ + union rte_event_dma_metadata *dma_mdata; + struct rte_event_dma_request *req_info; + const struct rte_dma_sge *fptr, *lptr; + struct rte_event_dma_adapter_op *op; + struct cnxk_dpi_compl_s *comp_ptr; + struct cnxk_dpi_conf *dpi_conf; + struct cnxk_dpi_vf_s *dpivf; + struct rte_event *rsp_info; + uint16_t nb_src, nb_dst; + struct rte_dma_dev *dev; + uint64_t hdr[4]; + uint16_t count; + int rc; + + PLT_SET_USED(ws); + + for (count = 0; count < nb_events; count++) { + op = ev[count].event_ptr; + dma_mdata = (union rte_event_dma_metadata *)((uint8_t *)op + + sizeof(struct rte_event_dma_adapter_op)); + rsp_info = &dma_mdata->response_info; + req_info = &dma_mdata->request_info; + dev = rte_dma_pmd_dev_get(req_info->dma_dev_id); + dpivf = dev->data->dev_private; + dpi_conf = &dpivf->conf[req_info->vchan]; + + if (unlikely(((dpi_conf->c_desc.tail + 1) & dpi_conf->c_desc.max_cnt) == + dpi_conf->c_desc.head)) + return count; + + comp_ptr = dpi_conf->c_desc.compl_ptr[dpi_conf->c_desc.tail]; + CNXK_DPI_STRM_INC(dpi_conf->c_desc, tail); + comp_ptr->op = op; + comp_ptr->dev_id = req_info->dma_dev_id; + comp_ptr->vchan = req_info->vchan; + comp_ptr->cdata = CNXK_DPI_REQ_SSO_CDATA; + + hdr[1] = dpi_conf->cmd.u | ((uint64_t)DPI_HDR_PT_WQP << 36); + hdr[2] = (uint64_t)comp_ptr; + + nb_src = op->nb_src & CNXK_DPI_MAX_POINTER; + nb_dst = op->nb_dst & CNXK_DPI_MAX_POINTER; + /* + * For inbound case, src pointers are last pointers. + * For all other cases, src pointers are first pointers. + */ + if (((dpi_conf->cmd.u >> 48) & DPI_HDR_XTYPE_MASK) == DPI_XTYPE_INBOUND) { + fptr = &op->dst_seg[0]; + lptr = &op->src_seg[0]; + RTE_SWAP(nb_src, nb_dst); + } else { + fptr = &op->src_seg[0]; + lptr = &op->dst_seg[0]; + } + + hdr[0] = ((uint64_t)nb_dst << 54) | (uint64_t)nb_src << 48; + hdr[0] |= (RTE_EVENT_TYPE_DMADEV << 28 | (rsp_info->sub_event_type << 20) | + rsp_info->flow_id); + hdr[0] |= ((uint64_t)(rsp_info->sched_type & DPI_HDR_TT_MASK)) << 32; + hdr[0] |= ((uint64_t)(rsp_info->queue_id & DPI_HDR_GRP_MASK)) << 34; + + rc = __dpi_queue_write_sg(dpivf, hdr, fptr, lptr, nb_src, nb_dst); + if (unlikely(rc)) { + CNXK_DPI_STRM_DEC(dpi_conf->c_desc, tail); + return rc; + } + + if (op->flags & RTE_DMA_OP_FLAG_SUBMIT) { + rte_wmb(); + plt_write64(dpi_conf->pnum_words + CNXK_DPI_CMD_LEN(nb_src, nb_dst), + dpivf->rdpi.rbase + DPI_VDMA_DBELL); + dpi_conf->stats.submitted += dpi_conf->pending + 1; + dpi_conf->pnum_words = 0; + dpi_conf->pending = 0; + } else { + dpi_conf->pnum_words += CNXK_DPI_CMD_LEN(nb_src, nb_dst); + dpi_conf->pending++; + } + } + + return count; +} + +uintptr_t +cnxk_dma_adapter_dequeue(uintptr_t get_work1) +{ + struct rte_event_dma_adapter_op *op; + struct cnxk_dpi_compl_s *comp_ptr; + struct cnxk_dpi_conf *dpi_conf; + struct cnxk_dpi_vf_s *dpivf; + struct rte_dma_dev *dev; + uint8_t *wqecs; + + comp_ptr = (struct cnxk_dpi_compl_s *)get_work1; + + /* Dequeue can be called without calling cnx_enqueue in case of + * dma_adapter. When its called from adapter, dma op will not be + * embedded in completion pointer. In those cases return op. + */ + if (comp_ptr->cdata != CNXK_DPI_REQ_SSO_CDATA) + return (uintptr_t)comp_ptr; + + dev = rte_dma_pmd_dev_get(comp_ptr->dev_id); + dpivf = dev->data->dev_private; + dpi_conf = &dpivf->conf[comp_ptr->vchan]; + + wqecs = (uint8_t *)&comp_ptr->wqecs; + if (__atomic_load_n(wqecs, __ATOMIC_RELAXED) != 0) + dpi_conf->stats.errors++; + + op = (struct rte_event_dma_adapter_op *)comp_ptr->op; + + /* We are done here. Reset completion buffer.*/ + comp_ptr->wqecs = ~0; + comp_ptr->op = NULL; + comp_ptr->dev_id = ~0; + comp_ptr->vchan = ~0; + comp_ptr->cdata = CNXK_DPI_REQ_CDATA; + + CNXK_DPI_STRM_INC(dpi_conf->c_desc, head); + /* Take into account errors also. This is similar to + * cnxk_dmadev_completed_status(). + */ + dpi_conf->stats.completed++; + + return (uintptr_t)op; +} diff --git a/drivers/dma/cnxk/meson.build b/drivers/dma/cnxk/meson.build index e557349368..9cf5453b0b 100644 --- a/drivers/dma/cnxk/meson.build +++ b/drivers/dma/cnxk/meson.build @@ -8,6 +8,10 @@ foreach flag: error_cflags endif endforeach -deps += ['bus_pci', 'common_cnxk', 'dmadev'] +driver_sdk_headers = files( + 'cnxk_dma_event_dp.h', +) + +deps += ['bus_pci', 'common_cnxk', 'dmadev', 'eventdev'] sources = files('cnxk_dmadev.c', 'cnxk_dmadev_fp.c') require_iova_in_mbuf = false diff --git a/drivers/dma/cnxk/version.map b/drivers/dma/cnxk/version.map new file mode 100644 index 0000000000..6cc1c6aaa5 --- /dev/null +++ b/drivers/dma/cnxk/version.map @@ -0,0 +1,9 @@ +INTERNAL { + global: + + cn10k_dma_adapter_enqueue; + cn9k_dma_adapter_enqueue; + cnxk_dma_adapter_dequeue; + + local: *; +}; -- 2.25.1