Add the data path functions for gathering completed operations
from IOAT devices.
Signed-off-by: Conor Walsh <conor.wa...@intel.com>
Signed-off-by: Kevin Laatz <kevin.la...@intel.com>
---
doc/guides/dmadevs/ioat.rst | 48 ++++++++++-
drivers/dma/ioat/ioat_dmadev.c | 141 +++++++++++++++++++++++++++++++++
2 files changed, 188 insertions(+), 1 deletion(-)
diff --git a/doc/guides/dmadevs/ioat.rst b/doc/guides/dmadevs/ioat.rst
index 16acfda3e3..42bbc0dee6 100644
--- a/doc/guides/dmadevs/ioat.rst
+++ b/doc/guides/dmadevs/ioat.rst
@@ -96,7 +96,9 @@ Performing Data Copies
To perform data copies using IOAT dmadev devices, the functions
``rte_dmadev_copy()`` and ``rte_dmadev_submit()`` should be used. Alternatively
``rte_dmadev_copy()`` can be called with the ``RTE_DMA_OP_FLAG_SUBMIT`` flag
-set.
+set. Once copies have been completed, the completion will be reported back when
+the application calls ``rte_dmadev_completed()`` or
+``rte_dmadev_completed_status()``.
The ``rte_dmadev_copy()`` function enqueues a single copy to the
device ring for copying at a later point. The parameters to the function
@@ -135,6 +137,50 @@ device and start the hardware processing of them:
return -1;
}
+To retrieve information about completed copies, the API
+``rte_dmadev_completed()`` or ``rte_dmadev_completed_status ()``should be
used.
+``rte_dmadev_completed()`` will return to the application how many operations
+have successfully completed upto a supplied maximum, it can also pass back the
+index of the last completed operation and if a completion within the requested
+batch has failed. If ``rte_dmadev_completed()`` passes back ``has_error`` as
+``true`` then ``rte_dmadev_completed_status()`` should be called to retrieve
+the detailed status of each completion in that batch. Once an error has
occurred
+``rte_dmadev_completed()`` will not process any more completions until
+``rte_dmadev_completed_status()`` has been called to get the error information.
+The supplied ``status`` array will contain either ``RTE_DMA_STATUS_SUCCESSFUL``
+if the operation was successful or a DMA error code if the operation failed.
+The status codes supported by IOAT are:
+
+* ``RTE_DMA_STATUS_SUCCESSFUL``: The operation was successful.
+* ``RTE_DMA_STATUS_INVALID_SRC_ADDR``: The operation failed due to an invalid
source address.
+* ``RTE_DMA_STATUS_INVALID_DST_ADDR``: The operation failed due to an invalid
destination address.
+* ``RTE_DMA_STATUS_INVALID_LENGTH``: The operation failed due to an invalid
descriptor length.
+* ``RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR``: The device could not read the
descriptor.
+* ``RTE_DMA_STATUS_ERROR_UNKNOWN``: The operation failed due to an unspecified
error.
+
+``rte_dmadev_completed_status()`` can be used on it's own instead of with
+``rte_dmadev_completed()`` but calling ``rte_dmadev_completed_status()`` may be
+slower than ``rte_dmadev_completed()``.
+
+The following code shows how to retrieve the number of successfully completed
+copies within a burst and then using ``rte_dmadev_completed_status()`` to check
+which operation failed and reset the device:
+
+.. code-block:: C
+
+ enum rte_dma_status_code status[COMP_BURST_SZ];
+ uint16_t count, idx, status_count;
+ bool error = 0;
+
+ count = rte_dmadev_completed(dev_id, vchan, COMP_BURST_SZ, &idx, &error);
+
+ if (error){
+ status_count = rte_dmadev_completed_status(dev_id, vchan, COMP_BURST_SZ,
&idx, status);
+ }
+
+After this snippet the application would use the status array to find which
copy
+failed and handle it appropriately.
+
Filling an Area of Memory
~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/drivers/dma/ioat/ioat_dmadev.c b/drivers/dma/ioat/ioat_dmadev.c
index 52c8db9f66..f765adcca6 100644
--- a/drivers/dma/ioat/ioat_dmadev.c
+++ b/drivers/dma/ioat/ioat_dmadev.c
@@ -6,6 +6,7 @@
#include <rte_dmadev_pmd.h>
#include <rte_malloc.h>
#include <rte_prefetch.h>
+#include <rte_errno.h>
#include "ioat_internal.h"
@@ -316,6 +317,144 @@ ioat_dev_dump(const struct rte_dmadev *dev, FILE *f)
return 0;
}
+/* Returns the index of the last completed operation. */
+static inline uint16_t
+__get_last_completed(struct ioat_dmadev *ioat, int *error)
+{
+ /* Status register contains the address of the completed operation */
+ uint64_t status = ioat->status;
+
+ /* lower 3 bits indicate "transfer status" : active, idle, halted.
+ * We can ignore bit 0.
+ */
+ *error = status & IOAT_CHANSTS_STATUS;
+
+ /* If we are just after recovering from an error the address returned by
+ * status will be 0, in this case we return the offset - 1 as the last
+ * completed. If not return the status value minus the chainaddr which
+ * gives us an offset into the ring. Right shifting by 6 (divide by 64)
+ * gives the index of the completion from the HW point of view and
adding
+ * the offset translates the ring index from HW to SW point of view.
+ */
+ if ((status & ~IOAT_CHANSTS_STATUS) == 0)
+ return ioat->offset - 1;
+
+ return (status - ioat->ring_addr) >> 6;
+}
+
+/* Translates IOAT ChanERRs to DMA error codes. */
+static inline enum rte_dma_status_code
+__translate_status_ioat_to_dma(uint32_t chanerr)
+{
+ if (chanerr & IOAT_CHANERR_INVALID_SRC_ADDR_MASK)
+ return RTE_DMA_STATUS_INVALID_SRC_ADDR;
+ else if (chanerr & IOAT_CHANERR_INVALID_DST_ADDR_MASK)
+ return RTE_DMA_STATUS_INVALID_DST_ADDR;
+ else if (chanerr & IOAT_CHANERR_INVALID_LENGTH_MASK)
+ return RTE_DMA_STATUS_INVALID_LENGTH;
+ else if (chanerr & IOAT_CHANERR_DESCRIPTOR_READ_ERROR_MASK)
+ return RTE_DMA_STATUS_DESCRIPTOR_READ_ERROR;
+ else
+ return RTE_DMA_STATUS_ERROR_UNKNOWN;
+}
+
+/* Returns details of operations that have been completed. */
+static uint16_t
+ioat_completed(struct rte_dmadev *dev, uint16_t qid __rte_unused, const
uint16_t max_ops,
+ uint16_t *last_idx, bool *has_error)
+{
+ struct ioat_dmadev *ioat = dev->dev_private;
+
+ const unsigned short mask = (ioat->qcfg.nb_desc - 1);
+ const unsigned short read = ioat->next_read;
+ unsigned short last_completed, count;
+ int error, fails = 0;
+
+ /* Do not do any work if there is an uncleared error. */
+ if (ioat->failure != 0) {
+ *has_error = true;
+ *last_idx = ioat->next_read - 2;
+ return 0;
+ }
+
+ last_completed = __get_last_completed(ioat, &error);
+ count = (last_completed + 1 - read) & mask;
+
+ /* Cap count at max_ops or set as last run in batch. */
+ if (count > max_ops)
+ count = max_ops;
+
+ if (count == max_ops || error != IOAT_CHANSTS_HALTED) {
+ ioat->next_read = read + count;
+ *last_idx = ioat->next_read - 1;
+ } else {
+ *has_error = true;
+ rte_errno = EIO;
+ ioat->failure = ioat->regs->chanerr;
+ ioat->next_read = read + count + 1;
+ if (__ioat_recover(ioat) != 0) {
+ IOAT_PMD_ERR("Device HALTED and could not be
recovered\n");
+ ioat_dev_dump(dev, stdout);
+ return 0;
+ }
+ __submit(ioat);
+ fails++;
+ *last_idx = ioat->next_read - 2;
+ }
+
+ return count;
+}
+
+/* Returns detailed status information about operations that have been
completed. */
+static uint16_t
+ioat_completed_status(struct rte_dmadev *dev, uint16_t qid __rte_unused,
+ uint16_t max_ops, uint16_t *last_idx, enum rte_dma_status_code
*status)
+{
+ struct ioat_dmadev *ioat = dev->dev_private;
+
+ const unsigned short mask = (ioat->qcfg.nb_desc - 1);
+ const unsigned short read = ioat->next_read;
+ unsigned short count, last_completed;
+ uint64_t fails = 0;
+ int error, i;
+
+ last_completed = __get_last_completed(ioat, &error);
+ count = (last_completed + 1 - read) & mask;
+
+ for (i = 0; i < RTE_MIN(count + 1, max_ops); i++)
+ status[i] = RTE_DMA_STATUS_SUCCESSFUL;
+
+ /* Cap count at max_ops or set as last run in batch. */
+ if (count > max_ops)
+ count = max_ops;
+
+ if (count == max_ops || error != IOAT_CHANSTS_HALTED)
+ ioat->next_read = read + count;
+ else {
+ rte_errno = EIO;
+ status[count] =
__translate_status_ioat_to_dma(ioat->regs->chanerr);
+ count++;
+ ioat->next_read = read + count;
+ if (__ioat_recover(ioat) != 0) {
+ IOAT_PMD_ERR("Device HALTED and could not be
recovered\n");
+ ioat_dev_dump(dev, stdout);
+ return 0;
+ }
+ __submit(ioat);
+ fails++;
+ }
+
+ if (ioat->failure > 0) {
+ status[0] = __translate_status_ioat_to_dma(ioat->failure);
+ count = RTE_MIN(count + 1, max_ops);
+ ioat->failure = 0;
+ }
+
+ *last_idx = ioat->next_read - 1;
+
+ return count;
+}
+
/* Create a DMA device. */
static int
ioat_dmadev_create(const char *name, struct rte_pci_device *dev)
@@ -356,6 +495,8 @@ ioat_dmadev_create(const char *name, struct rte_pci_device
*dev)
dmadev->dev_ops = &ioat_dmadev_ops;
+ dmadev->completed = ioat_completed;
+ dmadev->completed_status = ioat_completed_status;
dmadev->copy = ioat_enqueue_copy;
dmadev->fill = ioat_enqueue_fill;
dmadev->submit = ioat_submit;
--
2.25.1
