The current descriptor layout is:
struct dw_edma_desc *desc
└─ chunk list
└─ burst[]
Creating a DMA descriptor requires at least two kzalloc() calls because
each chunk is allocated as a linked-list node. Since the number of bursts
is already known when the descriptor is created, this linked-list layer is
unnecessary.
Move the burst array directly into struct dw_edma_desc and remove the
struct dw_edma_chunk layer entirely.
Use start_burst and done_burst to track the current bursts, which current
are in the DMA link list.
Signed-off-by: Frank Li <[email protected]>
---
drivers/dma/dw-edma/dw-edma-core.c | 130 ++++++++++++-------------------------
drivers/dma/dw-edma/dw-edma-core.h | 24 ++++---
2 files changed, 57 insertions(+), 97 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-core.c
b/drivers/dma/dw-edma/dw-edma-core.c
index
9e65155fd93d69ddbc8235fad671fad4dc120979..6e7b7ee99aaf6e1c3e354d36ca058813dc95b8dd
100644
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -40,76 +40,45 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan,
phys_addr_t cpu_addr)
return cpu_addr;
}
-static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc,
u32 nburst)
-{
- struct dw_edma_chan *chan = desc->chan;
- struct dw_edma_chunk *chunk;
-
- chunk = kzalloc(struct_size(chunk, burst, nburst), GFP_NOWAIT);
- if (unlikely(!chunk))
- return NULL;
-
- chunk->chan = chan;
- /* Toggling change bit (CB) in each chunk, this is a mechanism to
- * inform the eDMA HW block that this is a new linked list ready
- * to be consumed.
- * - Odd chunks originate CB equal to 0
- * - Even chunks originate CB equal to 1
- */
- chunk->cb = !(desc->chunks_alloc % 2);
-
- chunk->nburst = nburst;
-
- list_add_tail(&chunk->list, &desc->chunk_list);
- desc->chunks_alloc++;
-
- return chunk;
-}
-
-static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
+static struct dw_edma_desc *
+dw_edma_alloc_desc(struct dw_edma_chan *chan, u32 nburst)
{
struct dw_edma_desc *desc;
- desc = kzalloc(sizeof(*desc), GFP_NOWAIT);
+ desc = kzalloc(struct_size(desc, burst, nburst), GFP_NOWAIT);
if (unlikely(!desc))
return NULL;
desc->chan = chan;
-
- INIT_LIST_HEAD(&desc->chunk_list);
+ desc->nburst = nburst;
+ desc->cb = true;
return desc;
}
-static void dw_edma_free_desc(struct dw_edma_desc *desc)
-{
- struct dw_edma_chunk *child, *_next;
-
- /* Remove all the list elements */
- list_for_each_entry_safe(child, _next, &desc->chunk_list, list) {
- list_del(&child->list);
- kfree(child);
- desc->chunks_alloc--;
- }
-
- kfree(desc);
-}
-
static void vchan_free_desc(struct virt_dma_desc *vdesc)
{
- dw_edma_free_desc(vd2dw_edma_desc(vdesc));
+ kfree(vd2dw_edma_desc(vdesc));
}
-static void dw_edma_core_start(struct dw_edma_chunk *chunk, bool first)
+static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
{
- struct dw_edma_chan *chan = chunk->chan;
+ struct dw_edma_chan *chan = desc->chan;
u32 i = 0;
- for (i = 0; i < chunk->nburst; i++)
- dw_edma_core_ll_data(chan, &chunk->burst[i], i, chunk->cb,
- i == chunk->nburst - 1);
+ for (i = 0; i < desc->nburst; i++) {
+ if (i == chan->ll_max - 1)
+ break;
+
+ dw_edma_core_ll_data(chan, &desc->burst[i + desc->start_burst],
+ i, desc->cb,
+ i == desc->nburst - 1 || i == chan->ll_max
- 2);
+ }
- dw_edma_core_ll_link(chan, i, chunk->cb, chan->ll_region.paddr);
+ desc->done_burst = desc->start_burst;
+ desc->start_burst += i;
+
+ dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr);
if (first)
dw_edma_core_ch_enable(chan);
@@ -119,7 +88,6 @@ static void dw_edma_core_start(struct dw_edma_chunk *chunk,
bool first)
static int dw_edma_start_transfer(struct dw_edma_chan *chan)
{
- struct dw_edma_chunk *child;
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
@@ -131,16 +99,9 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
if (!desc)
return 0;
- child = list_first_entry_or_null(&desc->chunk_list,
- struct dw_edma_chunk, list);
- if (!child)
- return 0;
+ dw_edma_core_start(desc, !desc->start_burst);
- dw_edma_core_start(child, !desc->xfer_sz);
- desc->xfer_sz += child->xfer_sz;
- list_del(&child->list);
- kfree(child);
- desc->chunks_alloc--;
+ desc->cb = !desc->cb;
return 1;
}
@@ -289,8 +250,10 @@ dw_edma_device_tx_status(struct dma_chan *dchan,
dma_cookie_t cookie,
vd = vchan_find_desc(&chan->vc, cookie);
if (vd) {
desc = vd2dw_edma_desc(vd);
- if (desc)
- residue = desc->alloc_sz - desc->xfer_sz;
+
+ residue = desc->alloc_sz;
+ if (desc && desc->done_burst)
+ residue -= desc->burst[desc->done_burst].xfer_sz;
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
@@ -307,7 +270,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
enum dma_transfer_direction dir = xfer->direction;
struct scatterlist *sg = NULL;
- struct dw_edma_chunk *chunk = NULL;
struct dw_edma_burst *burst;
struct dw_edma_desc *desc;
u64 src_addr, dst_addr;
@@ -369,10 +331,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
return NULL;
}
- desc = dw_edma_alloc_desc(chan);
- if (unlikely(!desc))
- goto err_alloc;
-
if (xfer->type == EDMA_XFER_INTERLEAVED) {
src_addr = xfer->xfer.il->src_start;
dst_addr = xfer->xfer.il->dst_start;
@@ -396,19 +354,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
fsz = xfer->xfer.il->frame_size;
}
+ desc = dw_edma_alloc_desc(chan, cnt);
+ if (unlikely(!desc))
+ return NULL;
+
for (i = 0; i < cnt; i++) {
if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
break;
- if (!(i % chan->ll_max)) {
- u32 n = min(cnt - i, chan->ll_max);
-
- chunk = dw_edma_alloc_chunk(desc, n);
- if (unlikely(!chunk))
- goto err_alloc;
- }
-
- burst = chunk->burst + (i % chan->ll_max);
+ burst = desc->burst + i;
if (xfer->type == EDMA_XFER_CYCLIC)
burst->sz = xfer->xfer.cyclic.len;
@@ -417,8 +371,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
else if (xfer->type == EDMA_XFER_INTERLEAVED)
burst->sz = xfer->xfer.il->sgl[i % fsz].size;
- chunk->xfer_sz += burst->sz;
desc->alloc_sz += burst->sz;
+ burst->xfer_sz = desc->alloc_sz;
if (dir == DMA_DEV_TO_MEM) {
burst->sar = src_addr;
@@ -473,12 +427,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer,
}
return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
-
-err_alloc:
- if (desc)
- dw_edma_free_desc(desc);
-
- return NULL;
}
static struct dma_async_tx_descriptor *
@@ -551,8 +499,14 @@ static void dw_hdma_set_callback_result(struct
virt_dma_desc *vd,
return;
desc = vd2dw_edma_desc(vd);
- if (desc)
- residue = desc->alloc_sz - desc->xfer_sz;
+ residue = desc->alloc_sz;
+
+ if (desc) {
+ if (result == DMA_TRANS_NOERROR)
+ residue -= desc->burst[desc->start_burst - 1].xfer_sz;
+ else if (desc->done_burst)
+ residue -= desc->burst[desc->done_burst - 1].xfer_sz;
+ }
res = &vd->tx_result;
res->result = result;
@@ -571,7 +525,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan
*chan)
switch (chan->request) {
case EDMA_REQ_NONE:
desc = vd2dw_edma_desc(vd);
- if (!desc->chunks_alloc) {
+ if (desc->start_burst >= desc->nburst) {
dw_hdma_set_callback_result(vd,
DMA_TRANS_NOERROR);
list_del(&vd->node);
@@ -936,7 +890,7 @@ int dw_edma_probe(struct dw_edma_chip *chip)
goto err_irq_free;
/* Turn debugfs on */
- dw_edma_core_debugfs_on(dw);
+ //dw_edma_core_debugfs_on(dw);
chip->dw = dw;
diff --git a/drivers/dma/dw-edma/dw-edma-core.h
b/drivers/dma/dw-edma/dw-edma-core.h
index
1930c3bce2bf33fdfbf4e8d99002483a4565faed..ba83c42dee5224dccdf34cec6481e9404a607702
100644
--- a/drivers/dma/dw-edma/dw-edma-core.h
+++ b/drivers/dma/dw-edma/dw-edma-core.h
@@ -46,15 +46,8 @@ struct dw_edma_burst {
u64 sar;
u64 dar;
u32 sz;
-};
-
-struct dw_edma_chunk {
- struct list_head list;
- struct dw_edma_chan *chan;
- u8 cb;
+ /* precalulate summary of previous burst total size */
u32 xfer_sz;
- u32 nburst;
- struct dw_edma_burst burst[] __counted_by(nburst);
};
struct dw_edma_desc {
@@ -66,6 +59,12 @@ struct dw_edma_desc {
u32 alloc_sz;
u32 xfer_sz;
+
+ u32 done_burst;
+ u32 start_burst;
+ u8 cb;
+ u32 nburst;
+ struct dw_edma_burst burst[] __counted_by(nburst);
};
struct dw_edma_chan {
@@ -126,7 +125,6 @@ struct dw_edma_core_ops {
void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
void (*ch_doorbell)(struct dw_edma_chan *chan);
void (*ch_enable)(struct dw_edma_chan *chan);
-
void (*ch_config)(struct dw_edma_chan *chan);
void (*debugfs_on)(struct dw_edma *dw);
};
@@ -166,6 +164,14 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan
*dchan)
return vc2dw_edma_chan(to_virt_chan(dchan));
}
+static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan)
+{
+ if (chan->dir == EDMA_DIR_WRITE)
+ return chan->dw->chip->ll_region_wr[chan->id].paddr;
+
+ return chan->dw->chip->ll_region_rd[chan->id].paddr;
+}
+
static inline
void dw_edma_core_off(struct dw_edma *dw)
{
--
2.34.1
