Hi Jonathan Matthew, here is updated patch. could you review it?
---- while trying to add bio(4) support to mfii(4), we noticed that SAS2208 based card with cache memory hangs on boot. https://marc.info/?t=147738428600001&r=1&w=2 after investigation, we made conclusion that MPII SCSI PASSTHRU feature need to be used for sending MFI commands (such as DCMD_*) instead of using MFA request. this patch shouldn't change any functionality/behavior, but this is required for upcoming bio(4) support. tested on: SAS2208, SAS3008, and SAS3108 --- sys/dev/pci/mfii.c | 128 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 37 deletions(-) diff --git a/sys/dev/pci/mfii.c b/sys/dev/pci/mfii.c index 1f31bac..2a63a94 100644 --- a/sys/dev/pci/mfii.c +++ b/sys/dev/pci/mfii.c @@ -53,8 +53,6 @@ #define MFII_REQ_TYPE_NO_LOCK (0x2 << 1) #define MFII_REQ_TYPE_HI_PRI (0x6 << 1) -#define MFII_REQ_MFA(_a) htole64((_a) | MFII_REQ_TYPE_MFA) - #define MFII_FUNCTION_PASSTHRU_IO (0xf0) #define MFII_FUNCTION_LDIO_REQUEST (0xf1) @@ -67,15 +65,30 @@ #define MFII_CHAIN_FRAME_MIN 1024 -struct mfii_request_descr { +struct mfii_scsi_io_request_descr { u_int8_t flags; u_int8_t msix_index; u_int16_t smid; - u_int16_t lmid; u_int16_t dev_handle; } __packed; +struct mfii_mfa_io_request_descr { + u_int32_t flags:8; + u_int32_t addr1:24; + u_int32_t addr2; +} __packed; + +union mfii_request_descr { + struct mfii_scsi_io_request_descr scsi; + struct mfii_mfa_io_request_descr mfa; + struct { + u_int32_t lo; + u_int32_t hi; + } __packed u; + u_int64_t word; +}; + #define MFII_RAID_CTX_IO_TYPE_SYSPD (0x1 << 4) #define MFII_RAID_CTX_TYPE_CUDA (0x2 << 4) @@ -204,7 +217,7 @@ struct mfii_ccb { bus_addr_t ccb_sgl_offset; u_int ccb_sgl_len; - struct mfii_request_descr ccb_req; + union mfii_request_descr ccb_req; bus_dmamap_t ccb_dmamap; @@ -373,6 +386,7 @@ int mfii_mfa_poll(struct mfii_softc *, struct mfii_ccb *); int mfii_mgmt(struct mfii_softc *, struct mfii_ccb *, u_int32_t, const union mfi_mbox *, void *, size_t, int); +void mfii_empty_done(struct mfii_softc *, struct mfii_ccb *); int mfii_scsi_cmd_io(struct mfii_softc *, struct scsi_xfer *); @@ -918,8 +932,8 @@ mfii_dcmd_start(struct mfii_softc *sc, struct mfii_ccb *ccb) htolem32(&sge->sg_len, sizeof(*ccb->ccb_sense)); sge->sg_flags = MFII_SGE_CHAIN_ELEMENT | MFII_SGE_ADDR_IOCPLBNTA; - ccb->ccb_req.flags = MFII_REQ_TYPE_SCSI; - ccb->ccb_req.smid = letoh16(ccb->ccb_smid); + ccb->ccb_req.scsi.flags = MFII_REQ_TYPE_SCSI; + ccb->ccb_req.scsi.smid = letoh16(ccb->ccb_smid); mfii_start(sc, ccb); } @@ -1344,8 +1358,14 @@ mfii_get_info(struct mfii_softc *sc) int mfii_mfa_poll(struct mfii_softc *sc, struct mfii_ccb *ccb) { + /* + * Since the frame address passed to the F/W through Inbound Queue + * Port register must be aligned to 256 byte boundary, this means + * that we can not use MFI frame memory (ccb->ccb_mfi) which is 128 + * byte boundary here. Caller must ensure that the MFI frame is built + * in the MPII frame memory (ccb->ccb_request). + */ struct mfi_frame_header *hdr = ccb->ccb_request; - u_int64_t r; int to = 0, rv = 0; #ifdef DIAGNOSTIC @@ -1357,8 +1377,9 @@ mfii_mfa_poll(struct mfii_softc *sc, struct mfii_ccb *ccb) hdr->mfh_cmd_status = MFI_STAT_INVALID_STATUS; hdr->mfh_flags |= htole16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE); - r = MFII_REQ_MFA(ccb->ccb_request_dva); - memcpy(&ccb->ccb_req, &r, sizeof(ccb->ccb_req)); + ccb->ccb_req.u.lo = htole32(ccb->ccb_request_dva); + ccb->ccb_req.u.hi = htole32(ccb->ccb_request_dva >> 32); + ccb->ccb_req.mfa.flags |= MFII_REQ_TYPE_MFA; mfii_start(sc, ccb); @@ -1444,6 +1465,8 @@ mfii_exec(struct mfii_softc *sc, struct mfii_ccb *ccb) ccb->ccb_cookie = &m; ccb->ccb_done = mfii_exec_done; + mfii_start(sc, ccb); + mtx_enter(&m); while (ccb->ccb_cookie != NULL) msleep(ccb, &m, PRIBIO, "mfiiexec", 0); @@ -1468,11 +1491,13 @@ mfii_mgmt(struct mfii_softc *sc, struct mfii_ccb *ccb, u_int32_t opc, const union mfi_mbox *mbox, void *buf, size_t len, int flags) { - struct mfi_dcmd_frame *dcmd = ccb->ccb_request; - struct mfi_frame_header *hdr = &dcmd->mdf_header; - u_int64_t r; + struct mpii_msg_scsi_io *io = ccb->ccb_request; + struct mfii_raid_context *ctx = (struct mfii_raid_context *)(io + 1); + struct mfii_sge *sge = (struct mfii_sge *)(ctx + 1); + struct mfi_dcmd_frame *dcmd = ccb->ccb_mfi; + struct mfi_frame_header *hdr = &dcmd->mdf_header; u_int8_t *dma_buf; - int rv = EIO; + int rv = 0; dma_buf = dma_alloc(len, PR_WAITOK); if (dma_buf == NULL) @@ -1509,27 +1534,55 @@ mfii_mgmt(struct mfii_softc *sc, struct mfii_ccb *ccb, if (mbox != NULL) memcpy(&dcmd->mdf_mbox, mbox, sizeof(dcmd->mdf_mbox)); - if (ISSET(flags, SCSI_NOSLEEP)) - mfii_mfa_poll(sc, ccb); - else { - r = MFII_REQ_MFA(ccb->ccb_request_dva); - memcpy(&ccb->ccb_req, &r, sizeof(ccb->ccb_req)); + io->function = MFII_FUNCTION_PASSTHRU_IO; + io->sgl_offset0 = ((u_int8_t *)sge - (u_int8_t *)io) / 4; + io->chain_offset = ((u_int8_t *)sge - (u_int8_t *)io) / 16; + + htolem64(&sge->sg_addr, ccb->ccb_mfi_dva); + htolem32(&sge->sg_len, MFI_FRAME_SIZE); + sge->sg_flags = MFII_SGE_CHAIN_ELEMENT | MFII_SGE_ADDR_IOCPLBNTA; + + ccb->ccb_req.scsi.flags = MFII_REQ_TYPE_SCSI; + ccb->ccb_req.scsi.smid = letoh16(ccb->ccb_smid); + + if (ISSET(flags, SCSI_NOSLEEP)) { + /* busy-loop polling with done handler */ + ccb->ccb_cookie = NULL; + ccb->ccb_done = mfii_empty_done; + mfii_poll(sc, ccb); + } else { + /* sleep/wakeup without done handler */ + ccb->ccb_cookie = NULL; + ccb->ccb_done = NULL; mfii_exec(sc, ccb); } - if (hdr->mfh_cmd_status == MFI_STAT_OK) { - rv = 0; + if (ccb->ccb_len > 0) { + bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, + 0, ccb->ccb_dmamap->dm_mapsize, + (ccb->ccb_direction == MFII_DATA_IN) ? + BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); - if (ccb->ccb_direction == MFII_DATA_IN) - memcpy(buf, dma_buf, len); + bus_dmamap_unload(sc->sc_dmat, ccb->ccb_dmamap); } + rv = hdr->mfh_cmd_status == MFI_STAT_OK ? 0 : 1; + + if (rv == 0 && ccb->ccb_direction == MFII_DATA_IN) + memcpy(buf, dma_buf, len); + done: dma_free(dma_buf, len); return (rv); } +void +mfii_empty_done(struct mfii_softc *sc, struct mfii_ccb *ccb) +{ + return; +} + int mfii_load_mfa(struct mfii_softc *sc, struct mfii_ccb *ccb, void *sglp, int nosleep) @@ -1565,21 +1618,22 @@ mfii_load_mfa(struct mfii_softc *sc, struct mfii_ccb *ccb, void mfii_start(struct mfii_softc *sc, struct mfii_ccb *ccb) { - u_long *r = (u_long *)&ccb->ccb_req; - bus_dmamap_sync(sc->sc_dmat, MFII_DMA_MAP(sc->sc_requests), ccb->ccb_request_offset, MFII_REQUEST_SIZE, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); #if defined(__LP64__) - bus_space_write_raw_8(sc->sc_iot, sc->sc_ioh, MFI_IQPL, *r); + bus_space_write_raw_8(sc->sc_iot, sc->sc_ioh, + MFI_IQPL, ccb->ccb_req.word); #else mtx_enter(&sc->sc_post_mtx); - bus_space_write_raw_4(sc->sc_iot, sc->sc_ioh, MFI_IQPL, r[0]); + bus_space_write_raw_4(sc->sc_iot, sc->sc_ioh, + MFI_IQPL, ccb->ccb_req.u.lo); bus_space_barrier(sc->sc_iot, sc->sc_ioh, MFI_IQPL, 8, BUS_SPACE_BARRIER_WRITE); - bus_space_write_raw_4(sc->sc_iot, sc->sc_ioh, MFI_IQPH, r[1]); + bus_space_write_raw_4(sc->sc_iot, sc->sc_ioh, + MFI_IQPH, ccb->ccb_req.u.hi); bus_space_barrier(sc->sc_iot, sc->sc_ioh, MFI_IQPH, 8, BUS_SPACE_BARRIER_WRITE); mtx_leave(&sc->sc_post_mtx); @@ -1904,8 +1958,8 @@ mfii_scsi_cmd_io(struct mfii_softc *sc, struct scsi_xfer *xs) break; } - ccb->ccb_req.flags = sc->sc_iop->ldio_req_type; - ccb->ccb_req.smid = letoh16(ccb->ccb_smid); + ccb->ccb_req.scsi.flags = sc->sc_iop->ldio_req_type; + ccb->ccb_req.scsi.smid = letoh16(ccb->ccb_smid); return (0); } @@ -1951,8 +2005,8 @@ mfii_scsi_cmd_cdb(struct mfii_softc *sc, struct scsi_xfer *xs) ctx->num_sge = (ccb->ccb_len == 0) ? 0 : ccb->ccb_dmamap->dm_nsegs; - ccb->ccb_req.flags = MFII_REQ_TYPE_SCSI; - ccb->ccb_req.smid = letoh16(ccb->ccb_smid); + ccb->ccb_req.scsi.flags = MFII_REQ_TYPE_SCSI; + ccb->ccb_req.scsi.smid = letoh16(ccb->ccb_smid); return (0); } @@ -2072,9 +2126,9 @@ mfii_pd_scsi_cmd_cdb(struct mfii_softc *sc, struct scsi_xfer *xs) ctx->num_sge = (ccb->ccb_len == 0) ? 0 : ccb->ccb_dmamap->dm_nsegs; - ccb->ccb_req.flags = MFII_REQ_TYPE_HI_PRI; - ccb->ccb_req.smid = letoh16(ccb->ccb_smid); - ccb->ccb_req.dev_handle = dev_handle; + ccb->ccb_req.scsi.flags = MFII_REQ_TYPE_HI_PRI; + ccb->ccb_req.scsi.smid = letoh16(ccb->ccb_smid); + ccb->ccb_req.scsi.dev_handle = dev_handle; return (XS_NOERROR); } @@ -2218,8 +2272,8 @@ mfii_abort(struct mfii_softc *sc, struct mfii_ccb *accb, uint16_t dev_handle, htolem16(&req->task_mid, smid); msg->flags = flags; - accb->ccb_req.flags = MFII_REQ_TYPE_HI_PRI; - accb->ccb_req.smid = letoh16(accb->ccb_smid); + accb->ccb_req.scsi.flags = MFII_REQ_TYPE_HI_PRI; + accb->ccb_req.scsi.smid = letoh16(accb->ccb_smid); } void -- 2.7.4 -- FUKAUMI Naoki
