Author: mav Date: Tue Feb 27 01:48:13 2018 New Revision: 330049 URL: https://svnweb.freebsd.org/changeset/base/330049
Log: Allow physically non-contiguous chain frames allocation in mps(4)/mpr(4). Chain frames required to satisfy all 2K of declared I/Os of 128KB each take more then a megabyte of a physical memory, all of which existing code tries allocate as physically contiguous. This patch removes that physical contiguousness requirement, leaving only virtual contiguousness. I was thinking about other ways of allocation, but the less granular allocation becomes, the bigger is the overhead and/or complexity, reaching about 100% overhead if allocate each frame separately. The patch also bumps the chain frames hard limit from 2K to 16K. It is more than enough for the case of default REQ_FRAMES and MAXPHYS (the drivers will allocate less than that automatically), while in case of increased MAXPHYS it will control maximal memory usage. Sponsored by: iXsystems, Inc. Differential Revision: https://reviews.freebsd.org/D14420 Modified: head/sys/dev/mpr/mpr.c head/sys/dev/mpr/mprvar.h head/sys/dev/mps/mps.c head/sys/dev/mps/mpsvar.h Modified: head/sys/dev/mpr/mpr.c ============================================================================== --- head/sys/dev/mpr/mpr.c Tue Feb 27 01:36:43 2018 (r330048) +++ head/sys/dev/mpr/mpr.c Tue Feb 27 01:48:13 2018 (r330049) @@ -382,7 +382,7 @@ mpr_transition_operational(struct mpr_softc *sc) static void mpr_resize_queues(struct mpr_softc *sc) { - u_int reqcr, prireqcr, maxio, sges_per_frame; + u_int reqcr, prireqcr, maxio, sges_per_frame, chain_seg_size; /* * Size the queues. Since the reply queues always need one free @@ -413,15 +413,11 @@ mpr_resize_queues(struct mpr_softc *sc) * the size of an IEEE Simple SGE. */ if (sc->facts->MsgVersion >= MPI2_VERSION_02_05) { - sc->chain_seg_size = - htole16(sc->facts->IOCMaxChainSegmentSize); - if (sc->chain_seg_size == 0) { - sc->chain_frame_size = MPR_DEFAULT_CHAIN_SEG_SIZE * - MPR_MAX_CHAIN_ELEMENT_SIZE; - } else { - sc->chain_frame_size = sc->chain_seg_size * - MPR_MAX_CHAIN_ELEMENT_SIZE; - } + chain_seg_size = htole16(sc->facts->IOCMaxChainSegmentSize); + if (chain_seg_size == 0) + chain_seg_size = MPR_DEFAULT_CHAIN_SEG_SIZE; + sc->chain_frame_size = chain_seg_size * + MPR_MAX_CHAIN_ELEMENT_SIZE; } else { sc->chain_frame_size = sc->reqframesz; } @@ -766,11 +762,11 @@ mpr_iocfacts_free(struct mpr_softc *sc) if (sc->queues_dmat != NULL) bus_dma_tag_destroy(sc->queues_dmat); - if (sc->chain_busaddr != 0) + if (sc->chain_frames != NULL) { bus_dmamap_unload(sc->chain_dmat, sc->chain_map); - if (sc->chain_frames != NULL) bus_dmamem_free(sc->chain_dmat, sc->chain_frames, sc->chain_map); + } if (sc->chain_dmat != NULL) bus_dma_tag_destroy(sc->chain_dmat); @@ -1411,11 +1407,36 @@ mpr_alloc_replies(struct mpr_softc *sc) return (0); } +static void +mpr_load_chains_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct mpr_softc *sc = arg; + struct mpr_chain *chain; + bus_size_t bo; + int i, o, s; + + if (error != 0) + return; + + for (i = 0, o = 0, s = 0; s < nsegs; s++) { + for (bo = 0; bo + sc->chain_frame_size <= segs[s].ds_len; + bo += sc->chain_frame_size) { + chain = &sc->chains[i++]; + chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o); + chain->chain_busaddr = segs[s].ds_addr + bo; + o += sc->chain_frame_size; + mpr_free_chain(sc, chain); + } + if (bo != segs[s].ds_len) + o += segs[s].ds_len - bo; + } + sc->chain_free_lowwater = i; +} + static int mpr_alloc_requests(struct mpr_softc *sc) { struct mpr_command *cm; - struct mpr_chain *chain; int i, rsize, nsegs; rsize = sc->reqframesz * sc->num_reqs; @@ -1444,31 +1465,39 @@ mpr_alloc_requests(struct mpr_softc *sc) mpr_dprint(sc, MPR_INIT, "request frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->req_busaddr, rsize); + sc->chains = malloc(sizeof(struct mpr_chain) * sc->num_chains, M_MPR, + M_NOWAIT | M_ZERO); + if (!sc->chains) { + mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); + return (ENOMEM); + } rsize = sc->chain_frame_size * sc->num_chains; - if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ + if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 16, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - rsize, /* maxsize */ - 1, /* nsegments */ - rsize, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &sc->chain_dmat)) { + rsize, /* maxsize */ + howmany(rsize, PAGE_SIZE), /* nsegments */ + rsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->chain_dmat)) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain DMA tag\n"); return (ENOMEM); - } - if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, - BUS_DMA_NOWAIT, &sc->chain_map)) { + } + if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, + BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->chain_map)) { mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); return (ENOMEM); - } - bzero(sc->chain_frames, rsize); - bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize, - mpr_memaddr_cb, &sc->chain_busaddr, 0); - mpr_dprint(sc, MPR_INIT, "chain frames busaddr= %#016jx size= %d\n", - (uintmax_t)sc->chain_busaddr, rsize); + } + if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, + rsize, mpr_load_chains_cb, sc, BUS_DMA_NOWAIT)) { + mpr_dprint(sc, MPR_ERROR, "Cannot load chain memory\n"); + bus_dmamem_free(sc->chain_dmat, sc->chain_frames, + sc->chain_map); + return (ENOMEM); + } rsize = MPR_SENSE_LEN * sc->num_reqs; if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ @@ -1495,22 +1524,6 @@ mpr_alloc_requests(struct mpr_softc *sc) mpr_memaddr_cb, &sc->sense_busaddr, 0); mpr_dprint(sc, MPR_INIT, "sense frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->sense_busaddr, rsize); - - sc->chains = malloc(sizeof(struct mpr_chain) * sc->num_chains, M_MPR, - M_WAITOK | M_ZERO); - if (!sc->chains) { - mpr_dprint(sc, MPR_ERROR, "Cannot allocate chain memory\n"); - return (ENOMEM); - } - for (i = 0; i < sc->num_chains; i++) { - chain = &sc->chains[i]; - chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames + - i * sc->chain_frame_size); - chain->chain_busaddr = sc->chain_busaddr + - i * sc->chain_frame_size; - mpr_free_chain(sc, chain); - sc->chain_free_lowwater++; - } /* * Allocate NVMe PRP Pages for NVMe SGL support only if the FW supports Modified: head/sys/dev/mpr/mprvar.h ============================================================================== --- head/sys/dev/mpr/mprvar.h Tue Feb 27 01:36:43 2018 (r330048) +++ head/sys/dev/mpr/mprvar.h Tue Feb 27 01:48:13 2018 (r330049) @@ -41,7 +41,7 @@ #define MPR_PRI_REQ_FRAMES 128 #define MPR_EVT_REPLY_FRAMES 32 #define MPR_REPLY_FRAMES MPR_REQ_FRAMES -#define MPR_CHAIN_FRAMES 2048 +#define MPR_CHAIN_FRAMES 16384 #define MPR_MAXIO_PAGES (-1) #define MPR_SENSE_LEN SSD_FULL_SIZE #define MPR_MSI_MAX 1 @@ -322,7 +322,6 @@ struct mpr_softc { u_int maxio; int chain_free_lowwater; uint32_t chain_frame_size; - uint16_t chain_seg_size; int prp_buffer_size; int prp_pages_free; int prp_pages_free_lowwater; @@ -389,7 +388,6 @@ struct mpr_softc { bus_dmamap_t sense_map; uint8_t *chain_frames; - bus_addr_t chain_busaddr; bus_dma_tag_t chain_dmat; bus_dmamap_t chain_map; Modified: head/sys/dev/mps/mps.c ============================================================================== --- head/sys/dev/mps/mps.c Tue Feb 27 01:36:43 2018 (r330048) +++ head/sys/dev/mps/mps.c Tue Feb 27 01:48:13 2018 (r330049) @@ -743,11 +743,11 @@ mps_iocfacts_free(struct mps_softc *sc) if (sc->queues_dmat != NULL) bus_dma_tag_destroy(sc->queues_dmat); - if (sc->chain_busaddr != 0) + if (sc->chain_frames != NULL) { bus_dmamap_unload(sc->chain_dmat, sc->chain_map); - if (sc->chain_frames != NULL) bus_dmamem_free(sc->chain_dmat, sc->chain_frames, sc->chain_map); + } if (sc->chain_dmat != NULL) bus_dma_tag_destroy(sc->chain_dmat); @@ -1370,11 +1370,36 @@ mps_alloc_replies(struct mps_softc *sc) return (0); } +static void +mps_load_chains_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) +{ + struct mps_softc *sc = arg; + struct mps_chain *chain; + bus_size_t bo; + int i, o, s; + + if (error != 0) + return; + + for (i = 0, o = 0, s = 0; s < nsegs; s++) { + for (bo = 0; bo + sc->reqframesz <= segs[s].ds_len; + bo += sc->reqframesz) { + chain = &sc->chains[i++]; + chain->chain =(MPI2_SGE_IO_UNION *)(sc->chain_frames+o); + chain->chain_busaddr = segs[s].ds_addr + bo; + o += sc->reqframesz; + mps_free_chain(sc, chain); + } + if (bo != segs[s].ds_len) + o += segs[s].ds_len - bo; + } + sc->chain_free_lowwater = i; +} + static int mps_alloc_requests(struct mps_softc *sc) { struct mps_command *cm; - struct mps_chain *chain; int i, rsize, nsegs; rsize = sc->reqframesz * sc->num_reqs; @@ -1403,31 +1428,39 @@ mps_alloc_requests(struct mps_softc *sc) mps_dprint(sc, MPS_INIT, "request frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->req_busaddr, rsize); + sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2, + M_NOWAIT | M_ZERO); + if (!sc->chains) { + mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n"); + return (ENOMEM); + } rsize = sc->reqframesz * sc->num_chains; - if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ + if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ 16, 0, /* algnmnt, boundary */ BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - rsize, /* maxsize */ - 1, /* nsegments */ - rsize, /* maxsegsize */ - 0, /* flags */ - NULL, NULL, /* lockfunc, lockarg */ - &sc->chain_dmat)) { + rsize, /* maxsize */ + howmany(rsize, PAGE_SIZE), /* nsegments */ + rsize, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->chain_dmat)) { mps_dprint(sc, MPS_ERROR, "Cannot allocate chain DMA tag\n"); return (ENOMEM); - } - if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, - BUS_DMA_NOWAIT, &sc->chain_map)) { + } + if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, + BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->chain_map)) { mps_dprint(sc, MPS_ERROR, "Cannot allocate chain memory\n"); return (ENOMEM); - } - bzero(sc->chain_frames, rsize); - bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize, - mps_memaddr_cb, &sc->chain_busaddr, 0); - mps_dprint(sc, MPS_INIT, "chain frames busaddr= %#016jx size= %d\n", - (uintmax_t)sc->chain_busaddr, rsize); + } + if (bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, + rsize, mps_load_chains_cb, sc, BUS_DMA_NOWAIT)) { + mps_dprint(sc, MPS_ERROR, "Cannot load chain memory\n"); + bus_dmamem_free(sc->chain_dmat, sc->chain_frames, + sc->chain_map); + return (ENOMEM); + } rsize = MPS_SENSE_LEN * sc->num_reqs; if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ @@ -1454,22 +1487,6 @@ mps_alloc_requests(struct mps_softc *sc) mps_memaddr_cb, &sc->sense_busaddr, 0); mps_dprint(sc, MPS_INIT, "sense frames busaddr= %#016jx size= %d\n", (uintmax_t)sc->sense_busaddr, rsize); - - sc->chains = malloc(sizeof(struct mps_chain) * sc->num_chains, M_MPT2, - M_WAITOK | M_ZERO); - if(!sc->chains) { - mps_dprint(sc, MPS_ERROR, "Cannot allocate chains memory\n"); - return (ENOMEM); - } - for (i = 0; i < sc->num_chains; i++) { - chain = &sc->chains[i]; - chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames + - i * sc->reqframesz); - chain->chain_busaddr = sc->chain_busaddr + - i * sc->reqframesz; - mps_free_chain(sc, chain); - sc->chain_free_lowwater++; - } nsegs = (sc->maxio / PAGE_SIZE) + 1; if (bus_dma_tag_create( sc->mps_parent_dmat, /* parent */ Modified: head/sys/dev/mps/mpsvar.h ============================================================================== --- head/sys/dev/mps/mpsvar.h Tue Feb 27 01:36:43 2018 (r330048) +++ head/sys/dev/mps/mpsvar.h Tue Feb 27 01:48:13 2018 (r330049) @@ -43,7 +43,7 @@ #define MPS_PRI_REQ_FRAMES 128 #define MPS_EVT_REPLY_FRAMES 32 #define MPS_REPLY_FRAMES MPS_REQ_FRAMES -#define MPS_CHAIN_FRAMES 2048 +#define MPS_CHAIN_FRAMES 16384 #define MPS_MAXIO_PAGES (-1) #define MPS_SENSE_LEN SSD_FULL_SIZE #define MPS_MSI_MAX 1 @@ -378,7 +378,6 @@ struct mps_softc { bus_dmamap_t sense_map; uint8_t *chain_frames; - bus_addr_t chain_busaddr; bus_dma_tag_t chain_dmat; bus_dmamap_t chain_map; _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "[email protected]"
