Add support for the Host1x unit to be located behind an IOMMU. This is required when gather buffers may be allocated non-contiguously in physical memory, as can be the case when TegraDRM is also using the IOMMU.
Signed-off-by: Mikko Perttunen <mperttunen at nvidia.com> --- drivers/gpu/host1x/cdma.c | 65 +++++++++++++++++++++++++++++------ drivers/gpu/host1x/cdma.h | 4 ++- drivers/gpu/host1x/dev.c | 39 +++++++++++++++++++-- drivers/gpu/host1x/dev.h | 5 +++ drivers/gpu/host1x/hw/cdma_hw.c | 10 +++--- drivers/gpu/host1x/job.c | 76 +++++++++++++++++++++++++++++++++++------ drivers/gpu/host1x/job.h | 1 + 7 files changed, 171 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/host1x/cdma.c b/drivers/gpu/host1x/cdma.c index c5d82a8..14692446e 100644 --- a/drivers/gpu/host1x/cdma.c +++ b/drivers/gpu/host1x/cdma.c @@ -51,9 +51,15 @@ static void host1x_pushbuffer_destroy(struct push_buffer *pb) struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); - if (pb->phys != 0) - dma_free_wc(host1x->dev, pb->size_bytes + 4, pb->mapped, - pb->phys); + if (!pb->phys) + return; + + if (host1x->domain) { + iommu_unmap(host1x->domain, pb->dma, pb->alloc_size_bytes); + free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma)); + } + + dma_free_wc(host1x->dev, pb->alloc_size_bytes, pb->mapped, pb->phys); pb->mapped = NULL; pb->phys = 0; @@ -66,28 +72,65 @@ static int host1x_pushbuffer_init(struct push_buffer *pb) { struct host1x_cdma *cdma = pb_to_cdma(pb); struct host1x *host1x = cdma_to_host1x(cdma); + struct iova *alloc; + int err; pb->mapped = NULL; pb->phys = 0; pb->size_bytes = HOST1X_PUSHBUFFER_SLOTS * 8; + pb->alloc_size_bytes = pb->size_bytes + 4; /* initialize buffer pointers */ pb->fence = pb->size_bytes - 8; pb->pos = 0; - /* allocate and map pushbuffer memory */ - pb->mapped = dma_alloc_wc(host1x->dev, pb->size_bytes + 4, &pb->phys, - GFP_KERNEL); - if (!pb->mapped) - goto fail; + if (host1x->domain) { + unsigned long shift; + + pb->alloc_size_bytes = iova_align(&host1x->iova, + pb->alloc_size_bytes); + + pb->mapped = dma_alloc_wc(host1x->dev, pb->alloc_size_bytes, + &pb->phys, GFP_KERNEL); + if (!pb->mapped) + return -ENOMEM; + + shift = iova_shift(&host1x->iova); + alloc = alloc_iova( + &host1x->iova, pb->alloc_size_bytes >> shift, + host1x->domain->geometry.aperture_end >> shift, true); + if (!alloc) { + err = -ENOMEM; + goto iommu_free_mem; + } + + err = iommu_map(host1x->domain, + iova_dma_addr(&host1x->iova, alloc), + pb->phys, pb->alloc_size_bytes, + IOMMU_READ); + if (err) + goto iommu_free_iova; + + pb->dma = iova_dma_addr(&host1x->iova, alloc); + } else { + pb->mapped = dma_alloc_wc(host1x->dev, pb->alloc_size_bytes, + &pb->phys, GFP_KERNEL); + if (!pb->mapped) + return -ENOMEM; + + pb->dma = pb->phys; + } host1x_hw_pushbuffer_init(host1x, pb); return 0; -fail: - host1x_pushbuffer_destroy(pb); - return -ENOMEM; +iommu_free_iova: + __free_iova(&host1x->iova, alloc); +iommu_free_mem: + dma_free_wc(host1x->dev, pb->alloc_size_bytes, pb->mapped, pb->phys); + + return err; } /* diff --git a/drivers/gpu/host1x/cdma.h b/drivers/gpu/host1x/cdma.h index 470087a..4b3645f 100644 --- a/drivers/gpu/host1x/cdma.h +++ b/drivers/gpu/host1x/cdma.h @@ -43,10 +43,12 @@ struct host1x_job; struct push_buffer { void *mapped; /* mapped pushbuffer memory */ - dma_addr_t phys; /* physical address of pushbuffer */ + dma_addr_t dma; /* device address of pushbuffer */ + phys_addr_t phys; /* physical address of pushbuffer */ u32 fence; /* index we've written */ u32 pos; /* index to write to */ u32 size_bytes; + u32 alloc_size_bytes; }; struct buffer_timeout { diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c index a62317a..ca4527e 100644 --- a/drivers/gpu/host1x/dev.c +++ b/drivers/gpu/host1x/dev.c @@ -168,16 +168,36 @@ static int host1x_probe(struct platform_device *pdev) return err; } + if (iommu_present(&platform_bus_type)) { + struct iommu_domain_geometry *geometry; + unsigned long order; + + host->domain = iommu_domain_alloc(&platform_bus_type); + if (!host->domain) + return -ENOMEM; + + err = iommu_attach_device(host->domain, &pdev->dev); + if (err) + goto fail_free_domain; + + geometry = &host->domain->geometry; + + order = __ffs(host->domain->pgsize_bitmap); + init_iova_domain(&host->iova, 1UL << order, + geometry->aperture_start >> order, + geometry->aperture_end >> order); + } + err = host1x_channel_list_init(host); if (err) { dev_err(&pdev->dev, "failed to initialize channel list\n"); - return err; + goto fail_detach_device; } err = clk_prepare_enable(host->clk); if (err < 0) { dev_err(&pdev->dev, "failed to enable clock\n"); - return err; + goto fail_detach_device; } err = host1x_syncpt_init(host); @@ -206,6 +226,15 @@ static int host1x_probe(struct platform_device *pdev) host1x_syncpt_deinit(host); fail_unprepare_disable: clk_disable_unprepare(host->clk); +fail_detach_device: + if (host->domain) { + put_iova_domain(&host->iova); + iommu_detach_device(host->domain, &pdev->dev); + } +fail_free_domain: + if (host->domain) + iommu_domain_free(host->domain); + return err; } @@ -218,6 +247,12 @@ static int host1x_remove(struct platform_device *pdev) host1x_syncpt_deinit(host); clk_disable_unprepare(host->clk); + if (host->domain) { + put_iova_domain(&host->iova); + iommu_detach_device(host->domain, &pdev->dev); + iommu_domain_free(host->domain); + } + return 0; } diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h index 5220510..6189825 100644 --- a/drivers/gpu/host1x/dev.h +++ b/drivers/gpu/host1x/dev.h @@ -19,6 +19,8 @@ #include <linux/platform_device.h> #include <linux/device.h> +#include <linux/iommu.h> +#include <linux/iova.h> #include "channel.h" #include "syncpt.h" @@ -108,6 +110,9 @@ struct host1x { struct device *dev; struct clk *clk; + struct iommu_domain *domain; + struct iova_domain iova; + struct mutex intr_mutex; int intr_syncpt_irq; diff --git a/drivers/gpu/host1x/hw/cdma_hw.c b/drivers/gpu/host1x/hw/cdma_hw.c index 659c1bb..b8c0348 100644 --- a/drivers/gpu/host1x/hw/cdma_hw.c +++ b/drivers/gpu/host1x/hw/cdma_hw.c @@ -55,7 +55,7 @@ static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr, *(p++) = HOST1X_OPCODE_NOP; *(p++) = HOST1X_OPCODE_NOP; dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__, - &pb->phys, getptr); + &pb->dma, getptr); getptr = (getptr + 8) & (pb->size_bytes - 1); } @@ -78,9 +78,9 @@ static void cdma_start(struct host1x_cdma *cdma) HOST1X_CHANNEL_DMACTRL); /* set base, put and end pointer */ - host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); + host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT); - host1x_ch_writel(ch, cdma->push_buffer.phys + + host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size_bytes + 4, HOST1X_CHANNEL_DMAEND); @@ -115,8 +115,8 @@ static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr) HOST1X_CHANNEL_DMACTRL); /* set base, end pointer (all of memory) */ - host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART); - host1x_ch_writel(ch, cdma->push_buffer.phys + + host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART); + host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size_bytes, HOST1X_CHANNEL_DMAEND); diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c index a91b7c4..222d930 100644 --- a/drivers/gpu/host1x/job.c +++ b/drivers/gpu/host1x/job.c @@ -174,8 +174,9 @@ static int do_waitchks(struct host1x_job *job, struct host1x *host, return 0; } -static unsigned int pin_job(struct host1x_job *job) +static unsigned int pin_job(struct host1x *host, struct host1x_job *job) { + int err; unsigned int i; job->num_unpins = 0; @@ -186,12 +187,16 @@ static unsigned int pin_job(struct host1x_job *job) dma_addr_t phys_addr; reloc->target.bo = host1x_bo_get(reloc->target.bo); - if (!reloc->target.bo) + if (!reloc->target.bo) { + err = -EINVAL; goto unpin; + } phys_addr = host1x_bo_pin(reloc->target.bo, &sgt); - if (!phys_addr) + if (!phys_addr) { + err = -EINVAL; goto unpin; + } job->addr_phys[job->num_unpins] = phys_addr; job->unpins[job->num_unpins].bo = reloc->target.bo; @@ -204,25 +209,68 @@ static unsigned int pin_job(struct host1x_job *job) struct sg_table *sgt; dma_addr_t phys_addr; + size_t gather_size = 0; + struct scatterlist *sg; + struct iova *alloc; + unsigned long shift; + int j; + g->bo = host1x_bo_get(g->bo); - if (!g->bo) + if (!g->bo) { + err = -EINVAL; goto unpin; + } phys_addr = host1x_bo_pin(g->bo, &sgt); - if (!phys_addr) + if (!phys_addr) { + err = -EINVAL; goto unpin; + } + + if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { + for_each_sg(sgt->sgl, sg, sgt->nents, j) { + gather_size += sg->length; + } + gather_size = iova_align(&host->iova, gather_size); + + shift = iova_shift(&host->iova); + alloc = alloc_iova( + &host->iova, gather_size >> shift, + host->domain->geometry.aperture_end >> shift, + true); + if (!alloc) { + err = -ENOMEM; + goto unpin; + } + + err = iommu_map_sg(host->domain, + iova_dma_addr(&host->iova, alloc), + sgt->sgl, sgt->nents, IOMMU_READ); + if (err == 0) { + __free_iova(&host->iova, alloc); + err = -EINVAL; + goto unpin; + } + + job->addr_phys[job->num_unpins] = + iova_dma_addr(&host->iova, alloc); + job->unpins[job->num_unpins].size = gather_size; + } else { + job->addr_phys[job->num_unpins] = phys_addr; + } + + job->gather_addr_phys[i] = job->addr_phys[job->num_unpins]; - job->addr_phys[job->num_unpins] = phys_addr; job->unpins[job->num_unpins].bo = g->bo; job->unpins[job->num_unpins].sgt = sgt; job->num_unpins++; } - return job->num_unpins; + return 0; unpin: host1x_job_unpin(job); - return 0; + return err; } static int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf) @@ -525,8 +573,8 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev) host1x_syncpt_load(host->syncpt + i); /* pin memory */ - err = pin_job(job); - if (!err) + err = pin_job(host, job); + if (err) goto out; /* patch gathers */ @@ -569,11 +617,19 @@ EXPORT_SYMBOL(host1x_job_pin); void host1x_job_unpin(struct host1x_job *job) { + struct host1x *host = dev_get_drvdata(job->channel->dev->parent); unsigned int i; for (i = 0; i < job->num_unpins; i++) { struct host1x_job_unpin_data *unpin = &job->unpins[i]; + if (!IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && host->domain) { + iommu_unmap(host->domain, job->addr_phys[i], + unpin->size); + free_iova(&host->iova, + iova_pfn(&host->iova, job->addr_phys[i])); + } + host1x_bo_unpin(unpin->bo, unpin->sgt); host1x_bo_put(unpin->bo); } diff --git a/drivers/gpu/host1x/job.h b/drivers/gpu/host1x/job.h index 8b3c15d..878239c4 100644 --- a/drivers/gpu/host1x/job.h +++ b/drivers/gpu/host1x/job.h @@ -44,6 +44,7 @@ struct host1x_waitchk { struct host1x_job_unpin_data { struct host1x_bo *bo; struct sg_table *sgt; + size_t size; }; /* -- 2.10.2