Author: ian
Date: Wed Oct 21 15:06:48 2015
New Revision: 289701
URL: https://svnweb.freebsd.org/changeset/base/289701
Log:
Switch mips busdma to using the common busdma_buffalloc code. This amounts
to copying in some code from the armv4 busdma, and adapting a few variable
and flag names to match the surrounding mips code.
Instead of keeping a local cache of prealloced busdma_map structs on a
mutex-protected list, set up an uma zone to cache them.
Instead of all memory allocations using M_DEVBUF, use new categories
M_BUSDMA for allocations of metadata (tags, maps, segment tracking lists),
and M_BOUNCE for bounce pages.
When buffers are allocated out of the busdma_bufalloc zones the alignment
and size of the buffers is known, and the code can skip doing any "partial
cacheline flush" logic to preserve data that may be adjacent to the DMA
buffer but contain non-DMA data.
Reviewed by: adrian, imp
Modified:
head/sys/conf/files.mips
head/sys/mips/mips/busdma_machdep.c
Modified: head/sys/conf/files.mips
==============================================================================
--- head/sys/conf/files.mips Wed Oct 21 15:01:51 2015 (r289700)
+++ head/sys/conf/files.mips Wed Oct 21 15:06:48 2015 (r289701)
@@ -50,6 +50,7 @@ mips/mips/vm_machdep.c standard
# misc opt-in bits
kern/kern_clocksource.c standard
kern/link_elf_obj.c standard
+kern/subr_busdma_bufalloc.c standard
kern/subr_dummy_vdso_tc.c standard
kern/subr_sfbuf.c optional mips | mipsel | mipsn32
Modified: head/sys/mips/mips/busdma_machdep.c
==============================================================================
--- head/sys/mips/mips/busdma_machdep.c Wed Oct 21 15:01:51 2015
(r289700)
+++ head/sys/mips/mips/busdma_machdep.c Wed Oct 21 15:06:48 2015
(r289701)
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/malloc.h>
#include <sys/bus.h>
+#include <sys/busdma_bufalloc.h>
#include <sys/interrupt.h>
#include <sys/lock.h>
#include <sys/proc.h>
@@ -47,7 +48,10 @@ __FBSDID("$FreeBSD$");
#include <sys/sysctl.h>
#include <sys/uio.h>
+#include <vm/uma.h>
#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
@@ -129,9 +133,8 @@ static SYSCTL_NODE(_hw, OID_AUTO, busdma
SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
"Total bounce pages");
-#define DMAMAP_UNCACHEABLE 0x8
-#define DMAMAP_ALLOCATED 0x10
-#define DMAMAP_MALLOCUSED 0x20
+#define DMAMAP_UNCACHEABLE 0x08
+#define DMAMAP_CACHE_ALIGNED 0x10
struct bus_dmamap {
struct bp_list bpages;
@@ -153,16 +156,6 @@ struct bus_dmamap {
static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
-static TAILQ_HEAD(,bus_dmamap) dmamap_freelist =
- TAILQ_HEAD_INITIALIZER(dmamap_freelist);
-
-#define BUSDMA_STATIC_MAPS 128
-static struct bus_dmamap map_pool[BUSDMA_STATIC_MAPS];
-
-static struct mtx busdma_mtx;
-
-MTX_SYSINIT(busdma_mtx, &busdma_mtx, "busdma lock", MTX_DEF);
-
static void init_bounce_pages(void *dummy);
static int alloc_bounce_zone(bus_dma_tag_t dmat);
static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
@@ -176,6 +169,80 @@ static void free_bounce_page(bus_dma_tag
/* Default tag, as most drivers provide no parent tag. */
bus_dma_tag_t mips_root_dma_tag;
+static uma_zone_t dmamap_zone; /* Cache of struct bus_dmamap items */
+
+static busdma_bufalloc_t coherent_allocator; /* Cache of coherent buffers */
+static busdma_bufalloc_t standard_allocator; /* Cache of standard buffers */
+
+MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
+MALLOC_DEFINE(M_BOUNCE, "bounce", "busdma bounce pages");
+
+/*
+ * This is the ctor function passed to uma_zcreate() for the pool of dma maps.
+ * It'll need platform-specific changes if this code is copied.
+ */
+static int
+dmamap_ctor(void *mem, int size, void *arg, int flags)
+{
+ bus_dmamap_t map;
+ bus_dma_tag_t dmat;
+
+ map = (bus_dmamap_t)mem;
+ dmat = (bus_dma_tag_t)arg;
+
+ dmat->map_count++;
+
+ map->dmat = dmat;
+ map->flags = 0;
+ map->slist = NULL;
+ map->allocbuffer = NULL;
+ map->sync_count = 0;
+ STAILQ_INIT(&map->bpages);
+
+ return (0);
+}
+
+/*
+ * This is the dtor function passed to uma_zcreate() for the pool of dma maps.
+ * It may need platform-specific changes if this code is copied .
+ */
+static void
+dmamap_dtor(void *mem, int size, void *arg)
+{
+ bus_dmamap_t map;
+
+ map = (bus_dmamap_t)mem;
+
+ map->dmat->map_count--;
+}
+
+static void
+busdma_init(void *dummy)
+{
+
+ /* Create a cache of maps for bus_dmamap_create(). */
+ dmamap_zone = uma_zcreate("dma maps", sizeof(struct bus_dmamap),
+ dmamap_ctor, dmamap_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
+
+ /* Create a cache of buffers in standard (cacheable) memory. */
+ standard_allocator = busdma_bufalloc_create("buffer",
+ mips_pdcache_linesize, /* minimum_alignment */
+ NULL, /* uma_alloc func */
+ NULL, /* uma_free func */
+ 0); /* uma_zcreate_flags */
+
+ /*
+ * Create a cache of buffers in uncacheable memory, to implement the
+ * BUS_DMA_COHERENT flag.
+ */
+ coherent_allocator = busdma_bufalloc_create("coherent",
+ mips_pdcache_linesize, /* minimum_alignment */
+ busdma_bufalloc_alloc_uncacheable,
+ busdma_bufalloc_free_uncacheable,
+ 0); /* uma_zcreate_flags */
+}
+SYSINIT(busdma, SI_SUB_KMEM, SI_ORDER_FOURTH, busdma_init, NULL);
+
/*
* Return true if a match is made.
*
@@ -203,17 +270,6 @@ run_filter(bus_dma_tag_t dmat, bus_addr_
return (retval);
}
-static void
-mips_dmamap_freelist_init(void *dummy)
-{
- int i;
-
- for (i = 0; i < BUSDMA_STATIC_MAPS; i++)
- TAILQ_INSERT_HEAD(&dmamap_freelist, &map_pool[i], freelist);
-}
-
-SYSINIT(busdma, SI_SUB_VM, SI_ORDER_ANY, mips_dmamap_freelist_init, NULL);
-
/*
* Check to see if the specified page is in an allowed DMA range.
*/
@@ -277,24 +333,13 @@ _busdma_alloc_dmamap(bus_dma_tag_t dmat)
struct sync_list *slist;
bus_dmamap_t map;
- slist = malloc(sizeof(*slist) * dmat->nsegments, M_DEVBUF, M_NOWAIT);
+ slist = malloc(sizeof(*slist) * dmat->nsegments, M_BUSDMA, M_NOWAIT);
if (slist == NULL)
return (NULL);
- mtx_lock(&busdma_mtx);
- map = TAILQ_FIRST(&dmamap_freelist);
- if (map)
- TAILQ_REMOVE(&dmamap_freelist, map, freelist);
- mtx_unlock(&busdma_mtx);
- if (!map) {
- map = malloc(sizeof(*map), M_DEVBUF, M_NOWAIT | M_ZERO);
- if (map)
- map->flags = DMAMAP_ALLOCATED;
- } else
- map->flags = 0;
- if (map != NULL) {
- STAILQ_INIT(&map->bpages);
+ map = uma_zalloc_arg(dmamap_zone, dmat, M_NOWAIT);
+ if (map != NULL)
map->slist = slist;
- } else
+ else
free(slist, M_DEVBUF);
return (map);
}
@@ -303,13 +348,7 @@ static __inline void
_busdma_free_dmamap(bus_dmamap_t map)
{
free(map->slist, M_DEVBUF);
- if (map->flags & DMAMAP_ALLOCATED)
- free(map, M_DEVBUF);
- else {
- mtx_lock(&busdma_mtx);
- TAILQ_INSERT_HEAD(&dmamap_freelist, map, freelist);
- mtx_unlock(&busdma_mtx);
- }
+ uma_zfree(dmamap_zone, map);
}
/*
@@ -332,7 +371,7 @@ bus_dma_tag_create(bus_dma_tag_t parent,
if (!parent)
parent = mips_root_dma_tag;
- newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
+ newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_BUSDMA, M_NOWAIT);
if (newtag == NULL) {
CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
__func__, newtag, 0, error);
@@ -476,7 +515,7 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc(
- sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
+ sizeof(bus_dma_segment_t) * dmat->nsegments, M_BUSDMA,
M_NOWAIT);
if (dmat->segments == NULL) {
CTR3(KTR_BUSDMA, "%s: tag %p error %d",
@@ -491,10 +530,6 @@ bus_dmamap_create(bus_dma_tag_t dmat, in
return (ENOMEM);
}
*mapp = newmap;
- newmap->dmat = dmat;
- newmap->allocbuffer = NULL;
- newmap->sync_count = 0;
- dmat->map_count++;
/*
* Bouncing might be required if the driver asks for an active
@@ -565,7 +600,6 @@ bus_dmamap_destroy(bus_dma_tag_t dmat, b
}
if (dmat->bounce_zone)
dmat->bounce_zone->map_count--;
- dmat->map_count--;
_busdma_free_dmamap(map);
CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
return (0);
@@ -577,10 +611,14 @@ bus_dmamap_destroy(bus_dma_tag_t dmat, b
* A dmamap to for use with dmamap_load is also allocated.
*/
int
-bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
+bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddrp, int flags,
bus_dmamap_t *mapp)
{
bus_dmamap_t newmap = NULL;
+ busdma_bufalloc_t ba;
+ struct busdma_bufzone *bufzone;
+ vm_memattr_t memattr;
+ void *vaddr;
int mflags;
@@ -590,7 +628,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
mflags = M_WAITOK;
if (dmat->segments == NULL) {
dmat->segments = (bus_dma_segment_t *)malloc(
- sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
+ sizeof(bus_dma_segment_t) * dmat->nsegments, M_BUSDMA,
mflags);
if (dmat->segments == NULL) {
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
@@ -598,8 +636,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
return (ENOMEM);
}
}
- if (flags & BUS_DMA_ZERO)
- mflags |= M_ZERO;
newmap = _busdma_alloc_dmamap(dmat);
if (newmap == NULL) {
@@ -607,10 +643,6 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
__func__, dmat, dmat->flags, ENOMEM);
return (ENOMEM);
}
- dmat->map_count++;
- *mapp = newmap;
- newmap->dmat = dmat;
- newmap->sync_count = 0;
/*
* If all the memory is coherent with DMA then we don't need to
@@ -619,54 +651,62 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
if (dmat->flags & BUS_DMA_COHERENT)
flags &= ~BUS_DMA_COHERENT;
+ if (flags & BUS_DMA_COHERENT) {
+ memattr = VM_MEMATTR_UNCACHEABLE;
+ ba = coherent_allocator;
+ newmap->flags |= DMAMAP_UNCACHEABLE;
+ } else {
+ memattr = VM_MEMATTR_DEFAULT;
+ ba = standard_allocator;
+ }
+ /* All buffers we allocate are cache-aligned. */
+ newmap->flags |= DMAMAP_CACHE_ALIGNED;
+
+ if (flags & BUS_DMA_ZERO)
+ mflags |= M_ZERO;
+
/*
- * Allocate uncacheable memory if all else fails.
+ * Try to find a bufzone in the allocator that holds a cache of buffers
+ * of the right size for this request. If the buffer is too big to be
+ * held in the allocator cache, this returns NULL.
*/
- if (flags & BUS_DMA_COHERENT)
- newmap->flags |= DMAMAP_UNCACHEABLE;
+ bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
- if (dmat->maxsize <= PAGE_SIZE &&
- (dmat->alignment < dmat->maxsize) &&
- !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr) &&
- !(newmap->flags & DMAMAP_UNCACHEABLE)) {
- *vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
- newmap->flags |= DMAMAP_MALLOCUSED;
+ /*
+ * Allocate the buffer from the uma(9) allocator if...
+ * - It's small enough to be in the allocator (bufzone not NULL).
+ * - The alignment constraint isn't larger than the allocation size
+ * (the allocator aligns buffers to their size boundaries).
+ * - There's no need to handle lowaddr/highaddr exclusion zones.
+ * else allocate non-contiguous pages if...
+ * - The page count that could get allocated doesn't exceed nsegments.
+ * - The alignment constraint isn't larger than a page boundary.
+ * - There are no boundary-crossing constraints.
+ * else allocate a block of contiguous pages because one or more of the
+ * constraints is something that only the contig allocator can fulfill.
+ */
+ if (bufzone != NULL && dmat->alignment <= bufzone->size &&
+ !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr)) {
+ vaddr = uma_zalloc(bufzone->umazone, mflags);
+ } else if (dmat->nsegments >= btoc(dmat->maxsize) &&
+ dmat->alignment <= PAGE_SIZE && dmat->boundary == 0) {
+ vaddr = (void *)kmem_alloc_attr(kernel_arena, dmat->maxsize,
+ mflags, 0, dmat->lowaddr, memattr);
} else {
- /*
- * XXX Use Contigmalloc until it is merged into this facility
- * and handles multi-seg allocations. Nobody is doing
- * multi-seg allocations yet though.
- */
- *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
- 0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
- dmat->boundary);
- }
- if (*vaddr == NULL) {
- if (newmap != NULL) {
- _busdma_free_dmamap(newmap);
- dmat->map_count--;
- }
- *mapp = NULL;
- return (ENOMEM);
+ vaddr = (void *)kmem_alloc_contig(kernel_arena, dmat->maxsize,
+ mflags, 0, dmat->lowaddr, dmat->alignment, dmat->boundary,
+ memattr);
+ }
+ if (vaddr == NULL) {
+ _busdma_free_dmamap(newmap);
+ newmap = NULL;
+ } else {
+ newmap->sync_count = 0;
}
+ *vaddrp = vaddr;
+ *mapp = newmap;
- if (newmap->flags & DMAMAP_UNCACHEABLE) {
- void *tmpaddr = (void *)*vaddr;
-
- if (tmpaddr) {
- tmpaddr = (void *)pmap_mapdev(vtophys(tmpaddr),
- dmat->maxsize);
- newmap->origbuffer = *vaddr;
- newmap->allocbuffer = tmpaddr;
- mips_dcache_wbinv_range((vm_offset_t)*vaddr,
- dmat->maxsize);
- *vaddr = tmpaddr;
- } else
- newmap->origbuffer = newmap->allocbuffer = NULL;
- } else
- newmap->origbuffer = newmap->allocbuffer = NULL;
-
- return (0);
+ return (vaddr == NULL ? ENOMEM : 0);
}
/*
@@ -676,21 +716,24 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, voi
void
bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
{
- if (map->allocbuffer) {
- KASSERT(map->allocbuffer == vaddr,
- ("Trying to freeing the wrong DMA buffer"));
- vaddr = map->origbuffer;
- }
+ struct busdma_bufzone *bufzone;
+ busdma_bufalloc_t ba;
if (map->flags & DMAMAP_UNCACHEABLE)
- pmap_unmapdev((vm_offset_t)map->allocbuffer, dmat->maxsize);
- if (map->flags & DMAMAP_MALLOCUSED)
- free(vaddr, M_DEVBUF);
+ ba = coherent_allocator;
else
- contigfree(vaddr, dmat->maxsize, M_DEVBUF);
+ ba = standard_allocator;
- dmat->map_count--;
- _busdma_free_dmamap(map);
+ free(map->slist, M_DEVBUF);
+ uma_zfree(dmamap_zone, map);
+
+ bufzone = busdma_bufalloc_findzone(ba, dmat->maxsize);
+
+ if (bufzone != NULL && dmat->alignment <= bufzone->size &&
+ !_bus_dma_can_bounce(dmat->lowaddr, dmat->highaddr))
+ uma_zfree(bufzone->umazone, vaddr);
+ else
+ kmem_free(kernel_arena, (vm_offset_t)vaddr, dmat->maxsize);
CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
}
@@ -1013,7 +1056,7 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, b
}
static void
-bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op)
+bus_dmamap_sync_buf(vm_offset_t buf, int len, bus_dmasync_op_t op, int aligned)
{
char tmp_cl[mips_pdcache_linesize], tmp_clend[mips_pdcache_linesize];
vm_offset_t buf_cl, buf_clend;
@@ -1025,13 +1068,23 @@ bus_dmamap_sync_buf(vm_offset_t buf, int
* and could modify areas of memory that share the same cache line
* at the beginning and the ending of the buffer. In order to
* prevent a data loss we save these chunks in temporary buffer
- * before invalidation and restore them afer it
+ * before invalidation and restore them afer it.
+ *
+ * If the aligned flag is set the buffer came from our allocator caches
+ * which are always sized and aligned to cacheline boundaries, so we can
+ * skip preserving nearby data if a transfer is unaligned (especially
+ * it's likely to not end on a boundary).
*/
- buf_cl = buf & ~cache_linesize_mask;
- size_cl = buf & cache_linesize_mask;
- buf_clend = buf + len;
- size_clend = (mips_pdcache_linesize -
- (buf_clend & cache_linesize_mask)) & cache_linesize_mask;
+ if (aligned) {
+ size_cl = 0;
+ size_clend = 0;
+ } else {
+ buf_cl = buf & ~cache_linesize_mask;
+ size_cl = buf & cache_linesize_mask;
+ buf_clend = buf + len;
+ size_clend = (mips_pdcache_linesize -
+ (buf_clend & cache_linesize_mask)) & cache_linesize_mask;
+ }
switch (op) {
case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
@@ -1151,6 +1204,7 @@ void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
{
struct sync_list *sl, *end;
+ int aligned;
if (op == BUS_DMASYNC_POSTWRITE)
return;
@@ -1163,11 +1217,14 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus
if (map->flags & DMAMAP_UNCACHEABLE)
return;
+ aligned = (map->flags & DMAMAP_CACHE_ALIGNED) ? 1 : 0;
+
CTR3(KTR_BUSDMA, "%s: op %x flags %x", __func__, op, map->flags);
if (map->sync_count) {
end = &map->slist[map->sync_count];
for (sl = &map->slist[0]; sl != end; sl++)
- bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op);
+ bus_dmamap_sync_buf(sl->vaddr, sl->datacount, op,
+ aligned);
}
}
@@ -1209,7 +1266,7 @@ alloc_bounce_zone(bus_dma_tag_t dmat)
}
}
- if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
+ if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_BUSDMA,
M_NOWAIT | M_ZERO)) == NULL)
return (ENOMEM);
@@ -1280,12 +1337,12 @@ alloc_bounce_pages(bus_dma_tag_t dmat, u
while (numpages > 0) {
struct bounce_page *bpage;
- bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
+ bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_BUSDMA,
M_NOWAIT | M_ZERO);
if (bpage == NULL)
break;
- bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
+ bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_BOUNCE,
M_NOWAIT, 0ul,
bz->lowaddr,
PAGE_SIZE,
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
