The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=e43730e585fab3385c5b774b83e95e16c984dacc
commit e43730e585fab3385c5b774b83e95e16c984dacc Author: John Baldwin <[email protected]> AuthorDate: 2026-02-25 02:23:39 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2026-02-25 02:23:39 +0000 cxgbe tom: Use the same WRs as iSCSI to send PDUs for NVMe Reviewed by: np (earlier version) Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D55470 --- sys/dev/cxgbe/tom/t4_cpl_io.c | 112 +++++------------------------------------- 1 file changed, 13 insertions(+), 99 deletions(-) diff --git a/sys/dev/cxgbe/tom/t4_cpl_io.c b/sys/dev/cxgbe/tom/t4_cpl_io.c index 5c39ae5fa8f3..7e1c497240c2 100644 --- a/sys/dev/cxgbe/tom/t4_cpl_io.c +++ b/sys/dev/cxgbe/tom/t4_cpl_io.c @@ -496,9 +496,6 @@ t4_close_conn(struct adapter *sc, struct toepcb *toep) #define MIN_ISO_TX_CREDITS (howmany(sizeof(struct cpl_tx_data_iso), 16)) #define MIN_TX_CREDITS(iso) \ (MIN_OFLD_TX_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0)) -#define MIN_OFLD_TX_V2_CREDITS (howmany(sizeof(struct fw_ofld_tx_data_v2_wr) + 1, 16)) -#define MIN_TX_V2_CREDITS(iso) \ - (MIN_OFLD_TX_V2_CREDITS + ((iso) ? MIN_ISO_TX_CREDITS : 0)) _Static_assert(MAX_OFLD_TX_CREDITS <= MAX_OFLD_TX_SDESC_CREDITS, "MAX_OFLD_TX_SDESC_CREDITS too small"); @@ -546,46 +543,6 @@ max_dsgl_nsegs(int tx_credits, int iso) return (nseg); } -/* Maximum amount of immediate data we could stuff in a WR */ -static inline int -max_imm_payload_v2(int tx_credits, int iso) -{ - const int iso_cpl_size = iso ? sizeof(struct cpl_tx_data_iso) : 0; - - KASSERT(tx_credits >= 0 && - tx_credits <= MAX_OFLD_TX_CREDITS, - ("%s: %d credits", __func__, tx_credits)); - - if (tx_credits < MIN_TX_V2_CREDITS(iso)) - return (0); - - return (tx_credits * 16 - sizeof(struct fw_ofld_tx_data_v2_wr) - - iso_cpl_size); -} - -/* Maximum number of SGL entries we could stuff in a WR */ -static inline int -max_dsgl_nsegs_v2(int tx_credits, int iso, int imm_payload) -{ - int nseg = 1; /* ulptx_sgl has room for 1, rest ulp_tx_sge_pair */ - int sge_pair_credits = tx_credits - MIN_TX_V2_CREDITS(iso); - - KASSERT(tx_credits >= 0 && - tx_credits <= MAX_OFLD_TX_CREDITS, - ("%s: %d credits", __func__, tx_credits)); - - if (tx_credits < MIN_TX_V2_CREDITS(iso) || - sge_pair_credits <= howmany(imm_payload, 16)) - return (0); - sge_pair_credits -= howmany(imm_payload, 16); - - nseg += 2 * (sge_pair_credits * 16 / 24); - if ((sge_pair_credits * 16) % 24 == 16) - nseg++; - - return (nseg); -} - static inline void write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode, unsigned int immdlen, unsigned int plen, uint8_t credits, int shove, @@ -613,35 +570,6 @@ write_tx_wr(void *dst, struct toepcb *toep, int fw_wr_opcode, } } -static inline void -write_tx_v2_wr(void *dst, struct toepcb *toep, int fw_wr_opcode, - unsigned int immdlen, unsigned int plen, uint8_t credits, int shove, - int ulp_submode) -{ - struct fw_ofld_tx_data_v2_wr *txwr = dst; - uint32_t flags; - - memset(txwr, 0, sizeof(*txwr)); - txwr->op_to_immdlen = htobe32(V_WR_OP(fw_wr_opcode) | - V_FW_WR_IMMDLEN(immdlen)); - txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | - V_FW_WR_LEN16(credits)); - txwr->plen = htobe32(plen); - flags = V_TX_ULP_MODE(ULP_MODE_NVMET) | V_TX_ULP_SUBMODE(ulp_submode) | - V_TX_URG(0) | V_TX_SHOVE(shove); - - if (toep->params.tx_align > 0) { - if (plen < 2 * toep->params.emss) - flags |= F_FW_OFLD_TX_DATA_WR_LSODISABLE; - else - flags |= F_FW_OFLD_TX_DATA_WR_ALIGNPLD | - (toep->params.nagle == 0 ? 0 : - F_FW_OFLD_TX_DATA_WR_ALIGNPLDSHOVE); - } - - txwr->lsodisable_to_flags = htobe32(flags); -} - /* * Generate a DSGL from a starting mbuf. The total number of segments and the * maximum segments in any one mbuf are provided. @@ -1300,7 +1228,7 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) { struct mbuf *m; const struct nvme_tcp_common_pdu_hdr *hdr; - struct fw_v2_nvmet_tx_data_wr *txwr; + struct fw_ofld_tx_data_wr *txwr; struct cpl_tx_data_iso *cpl_iso; void *p; struct wrqe *wr; @@ -1330,29 +1258,16 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) return (wr); } - /* - * The first mbuf is the PDU header that is always sent as - * immediate data. - */ - imm_data = sndptr->m_len; - iso = mbuf_iscsi_iso(sndptr); - max_imm = max_imm_payload_v2(tx_credits, iso); - - /* - * Not enough credits for the PDU header. - */ - if (imm_data > max_imm) - return (NULL); - - max_nsegs = max_dsgl_nsegs_v2(tx_credits, iso, imm_data); + max_imm = max_imm_payload(tx_credits, iso); + max_nsegs = max_dsgl_nsegs(tx_credits, iso); iso_mss = mbuf_iscsi_iso_mss(sndptr); - plen = imm_data; + plen = 0; nsegs = 0; max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ nomap_mbuf_seen = false; - for (m = sndptr->m_next; m != NULL; m = m->m_next) { + for (m = sndptr; m != NULL; m = m->m_next) { int n; if (m->m_flags & M_EXTPG) @@ -1440,13 +1355,13 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) if (iso) wr_len += sizeof(struct cpl_tx_data_iso); if (plen <= max_imm && !nomap_mbuf_seen) { - /* Immediate data tx for full PDU */ + /* Immediate data tx */ imm_data = plen; wr_len += plen; nsegs = 0; } else { /* DSGL tx for PDU data */ - wr_len += roundup2(imm_data, 16); + imm_data = 0; wr_len += sizeof(struct ulptx_sgl) + ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; } @@ -1460,7 +1375,7 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) credits = howmany(wr->wr_len, 16); if (iso) { - write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR, + write_tx_wr(txwr, toep, FW_ISCSI_TX_DATA_WR, imm_data + sizeof(struct cpl_tx_data_iso), adjusted_plen, credits, shove, ulp_submode | ULP_ISO); cpl_iso = (struct cpl_tx_data_iso *)(txwr + 1); @@ -1470,16 +1385,15 @@ write_nvme_mbuf_wr(struct toepcb *toep, struct mbuf *sndptr) hdr->pdo); p = cpl_iso + 1; } else { - write_tx_v2_wr(txwr, toep, FW_V2_NVMET_TX_DATA_WR, imm_data, + write_tx_wr(txwr, toep, FW_OFLD_TX_DATA_WR, imm_data, adjusted_plen, credits, shove, ulp_submode); p = txwr + 1; } - /* PDU header (and immediate data payload). */ - m_copydata(sndptr, 0, imm_data, p); - if (nsegs != 0) { - p = roundup2((char *)p + imm_data, 16); - write_tx_sgl(p, sndptr->m_next, NULL, nsegs, max_nsegs_1mbuf); + if (imm_data != 0) { + m_copydata(sndptr, 0, plen, p); + } else { + write_tx_sgl(p, sndptr, NULL, nsegs, max_nsegs_1mbuf); if (wr_len & 0xf) { uint64_t *pad = (uint64_t *)((uintptr_t)txwr + wr_len); *pad = 0;
