The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=71d82199a111af67cba73e32a27a900d74c1a1cc
commit 71d82199a111af67cba73e32a27a900d74c1a1cc Author: John Baldwin <[email protected]> AuthorDate: 2025-11-10 15:50:48 +0000 Commit: John Baldwin <[email protected]> CommitDate: 2025-11-10 15:50:48 +0000 cxgbe: Add support to the base driver for NVMe/TCP PDU offload - Adds various per-queue counters similar to iSCSI PDU offload as well as a hook in the adapter softc for a reference to the NVMe/TCP softc. - Instruct the firmware to include a DDP indicator in the status field for NVMe/TCP PDU completion messages. This flag indicates if the payload data for a PDU has been received in the free list or if it was placed directly into a kernel I/O data buffer via DDP. Sponsored by: Chelsio Communications --- sys/dev/cxgbe/adapter.h | 11 ++++++++ sys/dev/cxgbe/t4_main.c | 16 ++++++++++++ sys/dev/cxgbe/t4_sge.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h index 57618b1ceb15..0946c3110817 100644 --- a/sys/dev/cxgbe/adapter.h +++ b/sys/dev/cxgbe/adapter.h @@ -723,6 +723,16 @@ struct sge_ofld_rxq { uint64_t rx_iscsi_padding_errors; uint64_t rx_iscsi_header_digest_errors; uint64_t rx_iscsi_data_digest_errors; + counter_u64_t rx_nvme_ddp_setup_ok; + counter_u64_t rx_nvme_ddp_setup_no_stag; + counter_u64_t rx_nvme_ddp_setup_error; + counter_u64_t rx_nvme_ddp_pdus; + counter_u64_t rx_nvme_ddp_octets; + counter_u64_t rx_nvme_fl_pdus; + counter_u64_t rx_nvme_fl_octets; + counter_u64_t rx_nvme_invalid_headers; + counter_u64_t rx_nvme_header_digest_errors; + counter_u64_t rx_nvme_data_digest_errors; uint64_t rx_aio_ddp_jobs; uint64_t rx_aio_ddp_octets; u_long rx_toe_tls_records; @@ -1000,6 +1010,7 @@ struct adapter { void *iwarp_softc; /* (struct c4iw_dev *) */ struct iw_tunables iwt; void *iscsi_ulp_softc; /* (struct cxgbei_data *) */ + void *nvme_ulp_softc; /* (struct nvmf_che_adapter *) */ struct l2t_data *l2t; /* L2 table */ struct smt_data *smt; /* Source MAC Table */ struct tid_info tids; diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c index 2f6eb6062c20..6133d810c003 100644 --- a/sys/dev/cxgbe/t4_main.c +++ b/sys/dev/cxgbe/t4_main.c @@ -13009,6 +13009,22 @@ clear_stats(struct adapter *sc, u_int port_id) ofld_rxq->rx_iscsi_ddp_octets = 0; ofld_rxq->rx_iscsi_fl_pdus = 0; ofld_rxq->rx_iscsi_fl_octets = 0; + counter_u64_zero( + ofld_rxq->rx_nvme_ddp_setup_ok); + counter_u64_zero( + ofld_rxq->rx_nvme_ddp_setup_no_stag); + counter_u64_zero( + ofld_rxq->rx_nvme_ddp_setup_error); + counter_u64_zero(ofld_rxq->rx_nvme_ddp_pdus); + counter_u64_zero(ofld_rxq->rx_nvme_ddp_octets); + counter_u64_zero(ofld_rxq->rx_nvme_fl_pdus); + counter_u64_zero(ofld_rxq->rx_nvme_fl_octets); + counter_u64_zero( + ofld_rxq->rx_nvme_invalid_headers); + counter_u64_zero( + ofld_rxq->rx_nvme_header_digest_errors); + counter_u64_zero( + ofld_rxq->rx_nvme_data_digest_errors); ofld_rxq->rx_aio_ddp_jobs = 0; ofld_rxq->rx_aio_ddp_octets = 0; ofld_rxq->rx_toe_tls_records = 0; diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c index 8cd5b0543d8a..4932a3abdeca 100644 --- a/sys/dev/cxgbe/t4_sge.c +++ b/sys/dev/cxgbe/t4_sge.c @@ -852,6 +852,11 @@ t4_tweak_chip_settings(struct adapter *sc) /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; + if (sc->nvmecaps != 0) { + /* Request DDP status bit for NVMe PDU completions. */ + m |= F_NVME_TCP_DDP_VAL_EN; + v |= F_NVME_TCP_DDP_VAL_EN; + } t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | @@ -4170,6 +4175,20 @@ alloc_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq, int idx, ofld_rxq->rx_iscsi_ddp_setup_ok = counter_u64_alloc(M_WAITOK); ofld_rxq->rx_iscsi_ddp_setup_error = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_ddp_setup_ok = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_ddp_setup_no_stag = + counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_ddp_setup_error = + counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_ddp_octets = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_ddp_pdus = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_fl_octets = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_fl_pdus = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_invalid_headers = counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_header_digest_errors = + counter_u64_alloc(M_WAITOK); + ofld_rxq->rx_nvme_data_digest_errors = + counter_u64_alloc(M_WAITOK); ofld_rxq->ddp_buffer_alloc = counter_u64_alloc(M_WAITOK); ofld_rxq->ddp_buffer_reuse = counter_u64_alloc(M_WAITOK); ofld_rxq->ddp_buffer_free = counter_u64_alloc(M_WAITOK); @@ -4207,6 +4226,16 @@ free_ofld_rxq(struct vi_info *vi, struct sge_ofld_rxq *ofld_rxq) MPASS(!(ofld_rxq->iq.flags & IQ_SW_ALLOCATED)); counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_ok); counter_u64_free(ofld_rxq->rx_iscsi_ddp_setup_error); + counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_ok); + counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_no_stag); + counter_u64_free(ofld_rxq->rx_nvme_ddp_setup_error); + counter_u64_free(ofld_rxq->rx_nvme_ddp_octets); + counter_u64_free(ofld_rxq->rx_nvme_ddp_pdus); + counter_u64_free(ofld_rxq->rx_nvme_fl_octets); + counter_u64_free(ofld_rxq->rx_nvme_fl_pdus); + counter_u64_free(ofld_rxq->rx_nvme_invalid_headers); + counter_u64_free(ofld_rxq->rx_nvme_header_digest_errors); + counter_u64_free(ofld_rxq->rx_nvme_data_digest_errors); counter_u64_free(ofld_rxq->ddp_buffer_alloc); counter_u64_free(ofld_rxq->ddp_buffer_reuse); counter_u64_free(ofld_rxq->ddp_buffer_free); @@ -4218,12 +4247,12 @@ static void add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, struct sge_ofld_rxq *ofld_rxq) { - struct sysctl_oid_list *children; + struct sysctl_oid_list *children, *top; if (ctx == NULL || oid == NULL) return; - children = SYSCTL_CHILDREN(oid); + top = children = SYSCTL_CHILDREN(oid); SYSCTL_ADD_U64(ctx, children, OID_AUTO, "rx_aio_ddp_jobs", CTLFLAG_RD, &ofld_rxq->rx_aio_ddp_jobs, 0, "# of aio_read(2) jobs completed via DDP"); @@ -4280,6 +4309,41 @@ add_ofld_rxq_sysctls(struct sysctl_ctx_list *ctx, struct sysctl_oid *oid, SYSCTL_ADD_U64(ctx, children, OID_AUTO, "data_digest_errors", CTLFLAG_RD, &ofld_rxq->rx_iscsi_data_digest_errors, 0, "# of PDUs with invalid data digests"); + + oid = SYSCTL_ADD_NODE(ctx, top, OID_AUTO, "nvme", + CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "TOE NVMe statistics"); + children = SYSCTL_CHILDREN(oid); + + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_ok", + CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_ok, + "# of times DDP buffer was setup successfully"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_no_stag", + CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_no_stag, + "# of times STAG was not available for DDP buffer setup"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_setup_error", + CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_setup_error, + "# of times DDP buffer setup failed"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_octets", + CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_octets, + "# of octets placed directly"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "ddp_pdus", + CTLFLAG_RD, &ofld_rxq->rx_nvme_ddp_pdus, + "# of PDUs with data placed directly"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_octets", + CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_octets, + "# of data octets delivered in freelist"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "fl_pdus", + CTLFLAG_RD, &ofld_rxq->rx_nvme_fl_pdus, + "# of PDUs with data delivered in freelist"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "invalid_headers", + CTLFLAG_RD, &ofld_rxq->rx_nvme_invalid_headers, + "# of PDUs with invalid header field"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "header_digest_errors", + CTLFLAG_RD, &ofld_rxq->rx_nvme_header_digest_errors, + "# of PDUs with invalid header digests"); + SYSCTL_ADD_COUNTER_U64(ctx, children, OID_AUTO, "data_digest_errors", + CTLFLAG_RD, &ofld_rxq->rx_nvme_data_digest_errors, + "# of PDUs with invalid data digests"); } #endif
