Implement the dmadev fast path (copy, submit, completed, completed_status and burst_capacity) for the AMD AE4DMA PMD and wire the entry points through fp_obj in ae4dma_dmadev_create().
The framework-visible ring index is a free-running 16-bit value; the HW ring slot is derived as (hw_base + idx) & (nb_desc - 1), which is why the vchan ring depth is kept a power of two. completed() and completed_status() always report the last completed ring_idx via last_idx, even when no new op is reaped, as required by the dmadev API. Only memory-to-memory copy is advertised, so fp_obj->fill is left zero-initialised. AE4DMA-specific completion handling: the engine reports completion by advancing the per-queue read_idx register rather than reliably writing a status word back, so the driver uses read_idx as the source of truth and only reads the descriptor err_code byte to flag failures. Signed-off-by: Raghavendra Ningoji <[email protected]> --- doc/guides/dmadevs/ae4dma.rst | 24 +++ drivers/dma/ae4dma/ae4dma_dmadev.c | 245 +++++++++++++++++++++++++++++ 2 files changed, 269 insertions(+) diff --git a/doc/guides/dmadevs/ae4dma.rst b/doc/guides/dmadevs/ae4dma.rst index eeed55c..10446ca 100644 --- a/doc/guides/dmadevs/ae4dma.rst +++ b/doc/guides/dmadevs/ae4dma.rst @@ -51,3 +51,27 @@ On probe the PMD performs the following steps for each PCI function: IOVA-contiguous memory, programs the queue base address and ring depth into the per-queue registers, and enables the queue. * Interrupts are masked; completion is polled by the application. + +Usage +----- + +Once a dmadev has been started, copies are submitted with +``rte_dma_copy()`` and completions are reaped with ``rte_dma_completed()`` +or ``rte_dma_completed_status()``. See the +:ref:`Enqueue / Dequeue API <dmadev_enqueue_dequeue>` section of the +dmadev library documentation for details. + +Limitations +----------- + +* Only memory-to-memory copies are supported. Fill, scatter-gather and + any other operation types are not advertised in + ``rte_dma_info::dev_capa``. +* The maximum number of descriptors per virtual channel is fixed by + hardware at 32, and the engine reserves one slot to distinguish a full + ring from an empty one, so at most 31 operations can be outstanding + (``rte_dma_burst_capacity()`` returns at most 31). The PMD rounds the + requested ring size up to a power of two and clamps it to 32. +* Only a single virtual channel per dmadev is supported; use the 16 + per-PCI-function dmadevs to obtain channel-level parallelism. +* Interrupt-driven completion is not supported. diff --git a/drivers/dma/ae4dma/ae4dma_dmadev.c b/drivers/dma/ae4dma/ae4dma_dmadev.c index 9910e11..8fcb0d5 100644 --- a/drivers/dma/ae4dma/ae4dma_dmadev.c +++ b/drivers/dma/ae4dma/ae4dma_dmadev.c @@ -162,6 +162,76 @@ ae4dma_dev_close(struct rte_dma_dev *dev) return 0; } +/* Ring the doorbell: publish all enqueued descriptors to the hardware. */ +static inline void +__submit(struct ae4dma_dmadev *ae4dma) +{ + struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q; + uint16_t nb = cmd_q->qcfg.nb_desc; + + if (nb == 0) + return; + + /* The HW producer index is the ring slot of the next free descriptor. */ + AE4DMA_WRITE_REG(&cmd_q->hwq_regs->write_idx, + (cmd_q->hw_base + cmd_q->next_write) & (nb - 1)); + cmd_q->stats.submitted += (uint16_t)(cmd_q->next_write - cmd_q->last_write); + cmd_q->last_write = cmd_q->next_write; +} + +static int +ae4dma_submit(void *dev_private, uint16_t vchan __rte_unused) +{ + struct ae4dma_dmadev *ae4dma = dev_private; + + __submit(ae4dma); + return 0; +} + +/* Write a copy descriptor; returns the free-running ring_idx assigned to it. */ +static inline int +__write_desc_copy(void *dev_private, rte_iova_t src, rte_iova_t dst, + uint32_t len, uint64_t flags) +{ + struct ae4dma_dmadev *ae4dma = dev_private; + struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q; + uint16_t ring_idx = cmd_q->next_write; + uint16_t nb = cmd_q->qcfg.nb_desc; + struct ae4dma_desc *dma_desc; + uint16_t in_flight; + + if (nb == 0) + return -EINVAL; + + /* Reserve one slot to distinguish full from empty on the ring. */ + in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read); + if (in_flight >= nb - 1) + return -ENOSPC; + + dma_desc = &cmd_q->qbase_desc[(cmd_q->hw_base + ring_idx) & (nb - 1)]; + memset(dma_desc, 0, sizeof(*dma_desc)); + dma_desc->length = len; + dma_desc->src_hi = upper_32_bits(src); + dma_desc->src_lo = lower_32_bits(src); + dma_desc->dst_hi = upper_32_bits(dst); + dma_desc->dst_lo = lower_32_bits(dst); + + cmd_q->next_write++; + if (flags & RTE_DMA_OP_FLAG_SUBMIT) + __submit(ae4dma); + + /* dmadev ring_idx is a free-running 16-bit value, not a ring slot. */ + return ring_idx; +} + +/* Enqueue a copy operation onto the ae4dma device. */ +static int +ae4dma_enqueue_copy(void *dev_private, uint16_t vchan __rte_unused, + rte_iova_t src, rte_iova_t dst, uint32_t length, uint64_t flags) +{ + return __write_desc_copy(dev_private, src, dst, length, flags); +} + static int ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f) { @@ -192,6 +262,174 @@ ae4dma_dev_dump(const struct rte_dma_dev *dev, FILE *f) return 0; } +/* Translate an AE4DMA descriptor error code to the dmadev status code. */ +static inline enum rte_dma_status_code +__translate_status_ae4dma_to_dma(enum ae4dma_dma_err status) +{ + switch (status) { + case AE4DMA_DMA_ERR_NO_ERR: + return RTE_DMA_STATUS_SUCCESSFUL; + case AE4DMA_DMA_ERR_INV_LEN: + return RTE_DMA_STATUS_INVALID_LENGTH; + case AE4DMA_DMA_ERR_INV_SRC: + return RTE_DMA_STATUS_INVALID_SRC_ADDR; + case AE4DMA_DMA_ERR_INV_DST: + return RTE_DMA_STATUS_INVALID_DST_ADDR; + default: + return RTE_DMA_STATUS_ERROR_UNKNOWN; + } +} + +/* + * Scan the HW queue for completed descriptors (non-blocking). + * + * The AE4DMA engine signals completion by advancing the per-queue + * read_idx register; it does not (reliably) write a status value back + * into the descriptor. We therefore use the HW read_idx register as the + * source of truth and only inspect the descriptor dw1.err_code byte to + * classify each completion as success or failure. + * + * next_read/next_write are free-running 16-bit indices; the HW ring slot + * is ((hw_base + idx) & mask) and the number of in-flight ops is + * (next_write - next_read), both relying on nb_desc being a power of two. + */ +static inline uint16_t +ae4dma_scan_hwq(struct ae4dma_cmd_queue *cmd_q, uint16_t max_ops, + uint16_t *failed_count) +{ + volatile struct ae4dma_desc *hw_desc; + uint16_t events_count = 0, fails = 0; + uint16_t nb = cmd_q->qcfg.nb_desc; + uint16_t hw_read_pos, tail_pos; + uint16_t newly_done, in_flight; + uint16_t scan_cap, mask; + + *failed_count = 0; + if (nb == 0) + return 0; + mask = nb - 1; + + in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read); + if (in_flight == 0) + return 0; + + /* Descriptors HW has consumed since our last visit: [tail, read). */ + hw_read_pos = (uint16_t)(AE4DMA_READ_REG(&cmd_q->hwq_regs->read_idx) & mask); + tail_pos = (uint16_t)((cmd_q->hw_base + cmd_q->next_read) & mask); + newly_done = (uint16_t)((hw_read_pos - tail_pos) & mask); + if (newly_done == 0) + return 0; + + scan_cap = newly_done; + if (scan_cap > in_flight) + scan_cap = in_flight; + if (scan_cap > max_ops) + scan_cap = max_ops; + + while (events_count < scan_cap) { + uint16_t slot = (cmd_q->hw_base + cmd_q->next_read + events_count) & mask; + uint8_t hw_status, hw_err; + + hw_desc = &cmd_q->qbase_desc[slot]; + hw_status = hw_desc->dw1.status; + hw_err = hw_desc->dw1.err_code; + + /* + * read_idx advancing is the definitive completion signal. + * The per-descriptor status byte is informational and may + * not yet be written when we observe it, so a slot is only + * a failure if the HW flagged DESC_ERROR or set err_code. + */ + if (hw_status == AE4DMA_DMA_DESC_ERROR || + hw_err != AE4DMA_DMA_ERR_NO_ERR) { + fails++; + AE4DMA_PMD_WARN("Desc failed: status=%u err=%u", + hw_status, hw_err); + } + cmd_q->status[events_count] = (enum ae4dma_dma_err)hw_err; + events_count++; + } + + cmd_q->next_read += events_count; + cmd_q->stats.completed += events_count; + cmd_q->stats.errors += fails; + *failed_count = fails; + return events_count; +} + +/* Returns successful operations count and sets error flag if any errors. */ +static uint16_t +ae4dma_completed(void *dev_private, uint16_t vchan __rte_unused, + const uint16_t max_ops, uint16_t *last_idx, bool *has_error) +{ + struct ae4dma_dmadev *ae4dma = dev_private; + struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q; + uint16_t err_count = 0; + uint16_t cpl_count; + + *has_error = false; + cpl_count = ae4dma_scan_hwq(cmd_q, max_ops, &err_count); + + /* + * last_idx always reports the ring_idx of the most recently completed + * op, even when this call reaps nothing (matches the dmadev contract + * and the ioat/idxd drivers). + */ + if (last_idx != NULL) + *last_idx = (uint16_t)(cmd_q->next_read - 1); + + if (err_count != 0) + *has_error = true; + + return cpl_count - err_count; +} + +static uint16_t +ae4dma_completed_status(void *dev_private, uint16_t vchan __rte_unused, + uint16_t max_ops, uint16_t *last_idx, + enum rte_dma_status_code *status) +{ + struct ae4dma_dmadev *ae4dma = dev_private; + struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q; + uint16_t err_count = 0; + uint16_t cpl_count; + uint16_t i; + + cpl_count = ae4dma_scan_hwq(cmd_q, max_ops, &err_count); + + if (last_idx != NULL) + *last_idx = (uint16_t)(cmd_q->next_read - 1); + + if (likely(err_count == 0)) { + for (i = 0; i < cpl_count; i++) + status[i] = RTE_DMA_STATUS_SUCCESSFUL; + } else { + for (i = 0; i < cpl_count; i++) + status[i] = __translate_status_ae4dma_to_dma(cmd_q->status[i]); + } + + return cpl_count; +} + +/* Get the remaining capacity of the ring. */ +static uint16_t +ae4dma_burst_capacity(const void *dev_private, uint16_t vchan __rte_unused) +{ + const struct ae4dma_dmadev *ae4dma = dev_private; + const struct ae4dma_cmd_queue *cmd_q = &ae4dma->cmd_q; + uint16_t nb = cmd_q->qcfg.nb_desc; + uint16_t in_flight; + + if (nb == 0) + return 0; + + /* One slot reserved to distinguish full from empty. */ + in_flight = (uint16_t)(cmd_q->next_write - cmd_q->next_read); + if (in_flight >= nb - 1) + return 0; + return (uint16_t)(nb - 1 - in_flight); +} + static int ae4dma_stats_get(const struct rte_dma_dev *dev, uint16_t vchan __rte_unused, struct rte_dma_stats *rte_stats, uint32_t size __rte_unused) @@ -339,6 +577,13 @@ ae4dma_dmadev_create(const char *name, struct rte_pci_device *dev, uint8_t qn) dmadev->fp_obj->dev_private = dmadev->data->dev_private; dmadev->dev_ops = &ae4dma_dmadev_ops; + dmadev->fp_obj->burst_capacity = ae4dma_burst_capacity; + dmadev->fp_obj->completed = ae4dma_completed; + dmadev->fp_obj->completed_status = ae4dma_completed_status; + dmadev->fp_obj->copy = ae4dma_enqueue_copy; + dmadev->fp_obj->submit = ae4dma_submit; + /* fill capability not advertised: leave fp_obj->fill as zero-initialised. */ + ae4dma = dmadev->data->dev_private; if (ae4dma_add_queue(ae4dma, dev, qn, name) != 0) -- 2.34.1
