Some DesignWare-based endpoints integrate an eDMA engine that can be programmed by the host via MMIO. The upcoming NTB transport remote-eDMA backend relies on this capability, but there is currently no upstream test coverage for the end-to-end control and data path.
Extend pci-epf-test with an optional remote eDMA test backend (built when CONFIG_DW_EDMA is enabled). - Reserve a spare BAR and expose a small 'pcitest_edma_info' header at BAR offset 0. The header carries a magic/version and describes the endpoint eDMA register window, per-direction linked-list (LL) locations and an endpoint test buffer. - Map the eDMA registers and LL locations into that BAR using BAR subrange mappings (address-match inbound iATU). To run this extra testing, two new endpoint commands are added: * COMMAND_REMOTE_EDMA_SETUP * COMMAND_REMOTE_EDMA_CHECKSUM When the former command is received, the endpoint prepares for the remote eDMA transfer. The CHECKSUM command is useful for Host-to-EP transfer testing, as the endpoint side is not expected to receive the DMA completion interrupt directly. Instead, the host asks the endpoint to compute a CRC32 over the transferred data. This backend is exercised by the host-side pci_endpoint_test driver via a new UAPI flag. Signed-off-by: Koichiro Den <[email protected]> --- drivers/pci/endpoint/functions/pci-epf-test.c | 477 ++++++++++++++++++ 1 file changed, 477 insertions(+) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index e560c3becebb..eea10bddcd2a 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -10,6 +10,7 @@ #include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/io.h> +#include <linux/iommu.h> #include <linux/module.h> #include <linux/msi.h> #include <linux/slab.h> @@ -33,6 +34,8 @@ #define COMMAND_COPY BIT(5) #define COMMAND_ENABLE_DOORBELL BIT(6) #define COMMAND_DISABLE_DOORBELL BIT(7) +#define COMMAND_REMOTE_EDMA_SETUP BIT(8) +#define COMMAND_REMOTE_EDMA_CHECKSUM BIT(9) #define STATUS_READ_SUCCESS BIT(0) #define STATUS_READ_FAIL BIT(1) @@ -48,6 +51,10 @@ #define STATUS_DOORBELL_ENABLE_FAIL BIT(11) #define STATUS_DOORBELL_DISABLE_SUCCESS BIT(12) #define STATUS_DOORBELL_DISABLE_FAIL BIT(13) +#define STATUS_REMOTE_EDMA_SETUP_SUCCESS BIT(14) +#define STATUS_REMOTE_EDMA_SETUP_FAIL BIT(15) +#define STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS BIT(16) +#define STATUS_REMOTE_EDMA_CHECKSUM_FAIL BIT(17) #define FLAG_USE_DMA BIT(0) @@ -77,6 +84,9 @@ struct pci_epf_test { bool dma_private; const struct pci_epc_features *epc_features; struct pci_epf_bar db_bar; + + /* For extended tests that rely on vendor-specific features */ + void *data; }; struct pci_epf_test_reg { @@ -117,6 +127,454 @@ static enum pci_barno pci_epf_test_next_free_bar(struct pci_epf_test *epf_test) return bar; } +#if IS_REACHABLE(CONFIG_DW_EDMA) +#include <linux/dma/edma.h> + +#define PCITEST_EDMA_INFO_MAGIC 0x414d4445U /* 'EDMA' */ +#define PCITEST_EDMA_INFO_VERSION 0x00010000U +#define PCITEST_EDMA_TEST_BUF_SIZE (1024 * 1024) + +struct pci_epf_test_edma { + /* Remote eDMA test resources */ + bool enabled; + enum pci_barno bar; + void *info; + size_t total_size; + void *test_buf; + dma_addr_t test_buf_phys; + size_t test_buf_size; + + /* DW eDMA specifics */ + phys_addr_t reg_phys; + size_t reg_submap_sz; + unsigned long reg_iova; + size_t reg_iova_sz; + phys_addr_t ll_rd_phys; + size_t ll_rd_sz_aligned; + phys_addr_t ll_wr_phys; + size_t ll_wr_sz_aligned; +}; + +struct pcitest_edma_info { + __le32 magic; + __le32 version; + + __le32 reg_off; + __le32 reg_size; + + __le64 ll_rd_phys; + __le32 ll_rd_off; + __le32 ll_rd_size; + + __le64 ll_wr_phys; + __le32 ll_wr_off; + __le32 ll_wr_size; + + __le64 test_buf_phys; + __le32 test_buf_size; +}; + +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test, + enum pci_barno barno) +{ + struct pci_epf_test_edma *edma = test->data; + + if (!edma) + return false; + + return barno == edma->bar; +} + +static void pci_epf_test_clear_submaps(struct pci_epf_bar *bar) +{ + kfree(bar->submap); + bar->submap = NULL; + bar->num_submap = 0; +} + +static int pci_epf_test_add_submap(struct pci_epf_bar *bar, phys_addr_t phys, + size_t size) +{ + struct pci_epf_bar_submap *submap, *new; + + new = krealloc_array(bar->submap, bar->num_submap + 1, sizeof(*new), + GFP_KERNEL); + if (!new) + return -ENOMEM; + + bar->submap = new; + submap = &bar->submap[bar->num_submap]; + submap->phys_addr = phys; + submap->size = size; + bar->num_submap++; + + return 0; +} + +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test) +{ + struct pci_epf_test_edma *edma = test->data; + struct pci_epf *epf = test->epf; + struct pci_epc *epc = epf->epc; + struct device *dev = epc->dev.parent; + struct iommu_domain *dom; + struct pci_epf_bar *bar; + enum pci_barno barno; + + if (!edma) + return; + + barno = edma->bar; + if (barno == NO_BAR) + return; + + bar = &epf->bar[barno]; + + dom = iommu_get_domain_for_dev(dev); + if (dom && edma->reg_iova_sz) { + iommu_unmap(dom, edma->reg_iova, edma->reg_iova_sz); + edma->reg_iova = 0; + edma->reg_iova_sz = 0; + } + + if (edma->test_buf) { + dma_free_coherent(dev, edma->test_buf_size, + edma->test_buf, + edma->test_buf_phys); + edma->test_buf = NULL; + edma->test_buf_phys = 0; + edma->test_buf_size = 0; + } + + if (edma->info) { + pci_epf_free_space(epf, edma->info, barno, PRIMARY_INTERFACE); + edma->info = NULL; + } + + pci_epf_test_clear_submaps(bar); + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, bar); + + edma->bar = NO_BAR; + edma->enabled = false; +} + +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test) +{ + const struct pci_epc_features *epc_features = test->epc_features; + struct pci_epf_test_edma *edma; + struct pci_epf *epf = test->epf; + struct pci_epc *epc = epf->epc; + struct pcitest_edma_info *info; + struct device *dev = epc->dev.parent; + struct dw_edma_region region; + struct iommu_domain *dom; + size_t reg_sz_aligned, ll_rd_sz_aligned, ll_wr_sz_aligned; + phys_addr_t phys, ll_rd_phys, ll_wr_phys; + size_t ll_rd_size, ll_wr_size; + resource_size_t reg_size; + unsigned long iova; + size_t off, size; + int ret; + + if (!test->dma_chan_tx || !test->dma_chan_rx) + return -ENODEV; + + edma = devm_kzalloc(&epf->dev, sizeof(*edma), GFP_KERNEL); + if (!edma) + return -ENOMEM; + test->data = edma; + + edma->bar = pci_epf_test_next_free_bar(test); + if (edma->bar == NO_BAR) { + dev_err(&epf->dev, "No spare BAR for remote eDMA (remote eDMA disabled)\n"); + ret = -ENOSPC; + goto err; + } + + ret = dw_edma_get_reg_window(epc, &edma->reg_phys, ®_size); + if (ret) { + dev_err(dev, "failed to get edma reg window: %d\n", ret); + goto err; + } + dom = iommu_get_domain_for_dev(dev); + if (dom) { + phys = edma->reg_phys & PAGE_MASK; + size = PAGE_ALIGN(reg_size + edma->reg_phys - phys); + iova = phys; + + ret = iommu_map(dom, iova, phys, size, + IOMMU_READ | IOMMU_WRITE | IOMMU_MMIO, + GFP_KERNEL); + if (ret) { + dev_err(dev, "failed to direct map eDMA reg: %d\n", ret); + goto err; + } + edma->reg_iova = iova; + edma->reg_iova_sz = size; + } + + /* Get LL location addresses and sizes */ + ret = dw_edma_chan_get_ll_region(test->dma_chan_rx, ®ion); + if (ret) { + dev_err(dev, "failed to get edma ll region for rx: %d\n", ret); + goto err; + } + ll_rd_phys = region.paddr; + ll_rd_size = region.sz; + + ret = dw_edma_chan_get_ll_region(test->dma_chan_tx, ®ion); + if (ret) { + dev_err(dev, "failed to get edma ll region for tx: %d\n", ret); + goto err; + } + ll_wr_phys = region.paddr; + ll_wr_size = region.sz; + + edma->test_buf_size = PCITEST_EDMA_TEST_BUF_SIZE; + edma->test_buf = dma_alloc_coherent(dev, edma->test_buf_size, + &edma->test_buf_phys, GFP_KERNEL); + if (!edma->test_buf) { + ret = -ENOMEM; + goto err; + } + + reg_sz_aligned = PAGE_ALIGN(reg_size); + ll_rd_sz_aligned = PAGE_ALIGN(ll_rd_size); + ll_wr_sz_aligned = PAGE_ALIGN(ll_wr_size); + edma->total_size = PAGE_SIZE + reg_sz_aligned + ll_rd_sz_aligned + + ll_wr_sz_aligned; + size = roundup_pow_of_two(edma->total_size); + + info = pci_epf_alloc_space(epf, size, edma->bar, + epc_features, PRIMARY_INTERFACE); + if (!info) { + ret = -ENOMEM; + goto err; + } + memset(info, 0, size); + + off = PAGE_SIZE; + info->magic = cpu_to_le32(PCITEST_EDMA_INFO_MAGIC); + info->version = cpu_to_le32(PCITEST_EDMA_INFO_VERSION); + + info->reg_off = cpu_to_le32(off); + info->reg_size = cpu_to_le32(reg_size); + off += reg_sz_aligned; + + info->ll_rd_phys = cpu_to_le64(ll_rd_phys); + info->ll_rd_off = cpu_to_le32(off); + info->ll_rd_size = cpu_to_le32(ll_rd_size); + off += ll_rd_sz_aligned; + + info->ll_wr_phys = cpu_to_le64(ll_wr_phys); + info->ll_wr_off = cpu_to_le32(off); + info->ll_wr_size = cpu_to_le32(ll_wr_size); + off += ll_wr_sz_aligned; + + info->test_buf_phys = cpu_to_le64(edma->test_buf_phys); + info->test_buf_size = cpu_to_le32(edma->test_buf_size); + + edma->info = info; + edma->reg_submap_sz = reg_sz_aligned; + edma->ll_rd_phys = ll_rd_phys; + edma->ll_wr_phys = ll_wr_phys; + edma->ll_rd_sz_aligned = ll_rd_sz_aligned; + edma->ll_wr_sz_aligned = ll_wr_sz_aligned; + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, + &epf->bar[edma->bar]); + if (ret) { + dev_err(dev, + "failed to init BAR%d for remote eDMA: %d\n", + edma->bar, ret); + goto err; + } + dev_info(dev, "BAR%d initialized for remote eDMA\n", edma->bar); + + return 0; + +err: + pci_epf_test_clean_remote_edma(test); + devm_kfree(&epf->dev, edma); + test->data = NULL; + return ret; +} + +static int pci_epf_test_map_remote_edma(struct pci_epf_test *test) +{ + struct pci_epf_test_edma *edma = test->data; + struct pcitest_edma_info *info; + struct pci_epf *epf = test->epf; + struct pci_epc *epc = epf->epc; + struct pci_epf_bar *bar; + enum pci_barno barno; + struct device *dev = epc->dev.parent; + int ret; + + if (!edma) + return -ENODEV; + + info = edma->info; + barno = edma->bar; + + if (barno == NO_BAR) + return -ENOSPC; + if (!info || !edma->test_buf) + return -ENODEV; + + bar = &epf->bar[barno]; + pci_epf_test_clear_submaps(bar); + + ret = pci_epf_test_add_submap(bar, bar->phys_addr, PAGE_SIZE); + if (ret) + return ret; + + ret = pci_epf_test_add_submap(bar, edma->reg_phys, edma->reg_submap_sz); + if (ret) + goto err_submap; + + ret = pci_epf_test_add_submap(bar, edma->ll_rd_phys, + edma->ll_rd_sz_aligned); + if (ret) + goto err_submap; + + ret = pci_epf_test_add_submap(bar, edma->ll_wr_phys, + edma->ll_wr_sz_aligned); + if (ret) + goto err_submap; + + if (bar->size > edma->total_size) { + ret = pci_epf_test_add_submap(bar, 0, + bar->size - edma->total_size); + if (ret) + goto err_submap; + } + + ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, bar); + if (ret) { + dev_err(dev, "failed to map BAR%d: %d\n", barno, ret); + goto err_submap; + } + + /* + * Endpoint-local interrupts must be ignored even if the host fails to + * mask them. + */ + ret = dw_edma_chan_irq_config(test->dma_chan_tx, DW_EDMA_CH_IRQ_REMOTE); + if (ret) { + dev_err(dev, "failed to set irq mode for tx channel: %d\n", + ret); + goto err_bar; + } + ret = dw_edma_chan_irq_config(test->dma_chan_rx, DW_EDMA_CH_IRQ_REMOTE); + if (ret) { + dev_err(dev, "failed to set irq mode for rx channel: %d\n", + ret); + goto err_bar; + } + + return 0; +err_bar: + pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[barno]); +err_submap: + pci_epf_test_clear_submaps(bar); + return ret; +} + +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test, + struct pci_epf_test_reg *reg) +{ + struct pci_epf_test_edma *edma = epf_test->data; + size_t size = le32_to_cpu(reg->size); + void *buf; + int ret; + + if (!edma || !edma->test_buf || size > edma->test_buf_size) { + reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL); + return; + } + + buf = edma->test_buf; + + if (!edma->enabled) { + /* NB. Currently DW eDMA is the only supported backend */ + ret = pci_epf_test_map_remote_edma(epf_test); + if (ret) { + WRITE_ONCE(reg->status, + cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL)); + return; + } + edma->enabled = true; + } + + /* Populate the test buffer with random data */ + get_random_bytes(buf, size); + reg->checksum = cpu_to_le32(crc32_le(~0, buf, size)); + + WRITE_ONCE(reg->status, cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_SUCCESS)); +} + +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test, + struct pci_epf_test_reg *reg) +{ + struct pci_epf_test_edma *edma = epf_test->data; + u32 status = le32_to_cpu(reg->status); + size_t size; + void *addr; + u32 crc32; + + size = le32_to_cpu(reg->size); + if (!edma || !edma->test_buf || size > edma->test_buf_size) { + status |= STATUS_REMOTE_EDMA_CHECKSUM_FAIL; + reg->status = cpu_to_le32(status); + return; + } + + addr = edma->test_buf; + crc32 = crc32_le(~0, addr, size); + status |= STATUS_REMOTE_EDMA_CHECKSUM_SUCCESS; + + reg->checksum = cpu_to_le32(crc32); + reg->status = cpu_to_le32(status); +} + +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan) +{ + dw_edma_chan_irq_config(chan, DW_EDMA_CH_IRQ_DEFAULT); +} +#else +static bool pci_epf_test_bar_is_reserved(struct pci_epf_test *test, + enum pci_barno barno) +{ + return false; +} + +static void pci_epf_test_clean_remote_edma(struct pci_epf_test *test) +{ +} + +static int pci_epf_test_init_remote_edma(struct pci_epf_test *test) +{ + return -EOPNOTSUPP; +} + +static void pci_epf_test_remote_edma_setup(struct pci_epf_test *epf_test, + struct pci_epf_test_reg *reg) +{ + reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_SETUP_FAIL); +} + +static void pci_epf_test_remote_edma_checksum(struct pci_epf_test *epf_test, + struct pci_epf_test_reg *reg) +{ + reg->status = cpu_to_le32(STATUS_REMOTE_EDMA_CHECKSUM_FAIL); +} + +static void pci_epf_test_reset_dma_chan(struct dma_chan *chan) +{ +} +#endif + static void pci_epf_test_dma_callback(void *param) { struct pci_epf_test *epf_test = param; @@ -168,6 +626,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, return -EINVAL; } + pci_epf_test_reset_dma_chan(chan); + if (epf_test->dma_private) { sconf.direction = dir; if (dir == DMA_MEM_TO_DEV) @@ -870,6 +1330,14 @@ static void pci_epf_test_cmd_handler(struct work_struct *work) pci_epf_test_disable_doorbell(epf_test, reg); pci_epf_test_raise_irq(epf_test, reg); break; + case COMMAND_REMOTE_EDMA_SETUP: + pci_epf_test_remote_edma_setup(epf_test, reg); + pci_epf_test_raise_irq(epf_test, reg); + break; + case COMMAND_REMOTE_EDMA_CHECKSUM: + pci_epf_test_remote_edma_checksum(epf_test, reg); + pci_epf_test_raise_irq(epf_test, reg); + break; default: dev_err(dev, "Invalid command 0x%x\n", command); break; @@ -961,6 +1429,10 @@ static int pci_epf_test_epc_init(struct pci_epf *epf) if (ret) epf_test->dma_supported = false; + ret = pci_epf_test_init_remote_edma(epf_test); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dev, "Remote eDMA setup failed\n"); + if (epf->vfunc_no <= 1) { ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); if (ret) { @@ -1007,6 +1479,7 @@ static void pci_epf_test_epc_deinit(struct pci_epf *epf) struct pci_epf_test *epf_test = epf_get_drvdata(epf); cancel_delayed_work_sync(&epf_test->cmd_handler); + pci_epf_test_clean_remote_edma(epf_test); pci_epf_test_clean_dma_chan(epf_test); pci_epf_test_clear_bar(epf); } @@ -1076,6 +1549,9 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf) if (bar == test_reg_bar) continue; + if (pci_epf_test_bar_is_reserved(epf_test, bar)) + continue; + if (epc_features->bar[bar].type == BAR_FIXED) test_reg_size = epc_features->bar[bar].fixed_size; else @@ -1146,6 +1622,7 @@ static void pci_epf_test_unbind(struct pci_epf *epf) cancel_delayed_work_sync(&epf_test->cmd_handler); if (epc->init_complete) { + pci_epf_test_clean_remote_edma(epf_test); pci_epf_test_clean_dma_chan(epf_test); pci_epf_test_clear_bar(epf); } -- 2.51.0
