The Rx path assumes every SGE response starts a new Free List buffer:
BUG_ON(!(len & F_RSPD_NEWBUF)); That is not always true. On T5/T6, small
packets can be packed into the same FL buffer. Only the first response for that
buffer has NEWBUF set; later responses can refer to the same buffer with a
different payload offset. The current PMD consumes one FL buffer per
response. When packed responses are delivered, the FL consumer state goes out
of sync with the hardware and the affected ingress queue stops making progress.
From user space this shows up as: - rx_bgN_dropped_packets increasing at
line rate - q_ipackets for the affected queue staying at 0 - imissed
increasing - no recovery until the port is restarted Fix this by tracking
packed-buffer alignment and copying packet payload from the current FL buffer
at q->offset, only freeing the FL buffer once it has actually been consumed.
This patch: - adds sge::fl_align from the ingress pad/pack settings -
introduces cxgbe_copy_rx_pkt() to copy from packed FL buffers - advances
q->offset by len rounded up to fl_align - frees an FL buffer only after full
consumption - drains the previous buffer when a later response arrives with
NEWBUF Reproduce (T62100-LP-CR, FW 2.1.19.0 / TP 0.1.23.2): dpdk-testpmd
-l 1-9 -a 0000:18:00.4 -- \ --rxq=32 --txq=32 --nb-cores=8
--forward-mode=rxonly -i testpmd> port stop all testpmd> flow flush 0
testpmd> flow create 0 ingress pattern eth \ / vlan tci spec 0 tci
mask 0x0007 \ / end actions queue index 5 / end testpmd> port
start all testpmd> start Without this patch, rx_qN_packets stays at 0 while
rx_bgN_dropped_packets rises at line rate. With the patch, rx_qN_packets tracks
received traffic and rx_bgN_dropped_packets stays at 0. Signed-off-by:
Abdulrahman Alshawi <[email protected]> ---
drivers/net/cxgbe/base/adapter.h | 1 + drivers/net/cxgbe/sge.c |
122 ++++++++++++++++++++++++------- 2 files changed, 98 insertions(+), 25
deletions(-) diff --git a/drivers/net/cxgbe/base/adapter.h
b/drivers/net/cxgbe/base/adapter.h index 207f3ecb88..e67cf22950 100644 ---
a/drivers/net/cxgbe/base/adapter.h +++ b/drivers/net/cxgbe/base/adapter.h @@
-280,6 +280,7 @@ struct sge { u16 max_ethqsets; /* # of available
Ethernet queue sets */ u32 stat_len; /* length of status page
at ring end */ u32 pktshift; /* padding between CPL & packet
data */ + u32 fl_align; /* packed Rx packet alignment */
/* response queue interrupt parameters */ u16 timer_val[SGE_NTIMERS]; diff
--git a/drivers/net/cxgbe/sge.c b/drivers/net/cxgbe/sge.c index
e9d45f24c4..7cf5c70775 100644 --- a/drivers/net/cxgbe/sge.c +++
b/drivers/net/cxgbe/sge.c @@ -1492,6 +1492,90 @@ static inline void
cxgbe_fill_mbuf_info(struct adapter *adap,
RTE_MBUF_F_RX_L4_CKSUM_BAD); } +static int cxgbe_copy_rx_pkt(struct sge_rspq
*q, struct sge_eth_rxq *rxq, + u32 len, struct rte_mbuf **out) +{
+ struct sge *s = &q->adapter->sge; + struct rte_mbuf *pkt; + char *dst;
+ u32 copied = 0; + u32 remaining = len; + + pkt =
rte_pktmbuf_alloc(q->mb_pool); + if (unlikely(!pkt)) { +
rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++; + rxq->stats.rx_drops++;
+ return -ENOMEM; + } + + dst = rte_pktmbuf_append(pkt, len); + if
(unlikely(!dst)) { + rte_pktmbuf_free(pkt); +
rxq->stats.rx_drops++; + return -ENOMEM; + } + + while (remaining) {
+ const struct rx_sw_desc *rsd = &rxq->fl.sdesc[rxq->fl.cidx]; +
struct rte_mbuf *src = rsd->buf; + u32 bufsz = get_buf_size(q->adapter,
rsd); + u32 copy_len; + + if (unlikely(!src || q->offset < 0 || +
(u32)q->offset >= bufsz)) { + rte_pktmbuf_free(pkt); +
rxq->stats.rx_drops++; + return -EINVAL; + } + +
copy_len = RTE_MIN(bufsz - (u32)q->offset, remaining); + rte_memcpy(dst +
copied, + rte_pktmbuf_mtod_offset(src, const void *, q->offset), +
copy_len); + + copied += copy_len; + remaining -=
copy_len; + q->offset += copy_len; + + if (remaining) { +
free_rx_bufs(&rxq->fl, 1); + q->offset = 0; + continue; +
} + + q->offset = RTE_ALIGN_CEIL(q->offset, s->fl_align); + if
((u32)q->offset >= bufsz) { + free_rx_bufs(&rxq->fl, 1); +
q->offset = 0; + } + } + + *out = pkt; + return 0; +} + +static
unsigned int cxgbe_fl_pkt_align(struct adapter *adap) +{ + u32 sge_control =
t4_read_reg(adap, A_SGE_CONTROL); + unsigned int ingpad_shift, ingpad,
fl_align; + + ingpad_shift = CHELSIO_CHIP_VERSION(adap->params.chip) <=
CHELSIO_T5 ? + X_INGPADBOUNDARY_SHIFT : X_T6_INGPADBOUNDARY_SHIFT;
+ ingpad = 1U << (G_INGPADBOUNDARY(sge_control) + ingpad_shift); + fl_align
= ingpad; + + if (!is_t4(adap->params.chip)) { + u32 sge_control2 =
t4_read_reg(adap, A_SGE_CONTROL2); + unsigned int ingpack =
G_INGPACKBOUNDARY(sge_control2); + + ingpack = ingpack ==
X_INGPACKBOUNDARY_16B ? + 16 : 1U << (ingpack +
X_INGPACKBOUNDARY_SHIFT); + fl_align = RTE_MAX(ingpad, ingpack); + } +
+ return fl_align ? fl_align : RTE_CACHE_LINE_SIZE; +} + /** *
process_responses - process responses from an SGE response queue * @q: the
ingress queue to process @@ -1535,14 +1619,12 @@ static int
process_responses(struct sge_rspq *q, int budget, stat_pidx =
ntohs(q->stat->pidx); stat_pidx_diff = P_IDXDIFF(q, stat_pidx);
while (stat_pidx_diff && budget_left) { - const struct
rx_sw_desc *rsd = - &rxq->fl.sdesc[rxq->fl.cidx];
const struct rss_header *rss_hdr = (const void
*)q->cur_desc; const struct cpl_rx_pkt *cpl =
(const void *)&q->cur_desc[1]; - struct rte_mbuf *pkt, *npkt; -
u32 len, bufsz; + struct rte_mbuf *pkt; +
u32 len; rc = (const struct rsp_ctrl *)
((const char *)q->cur_desc + @@ -1553,28 +1635,16 @@ static int
process_responses(struct sge_rspq *q, int budget, break;
len = ntohl(rc->pldbuflen_qid); - BUG_ON(!(len &
F_RSPD_NEWBUF)); - pkt = rsd->buf; - npkt = pkt; -
len = G_RSPD_LEN(len); - pkt->pkt_len = len; - -
/* Chain mbufs into len if necessary */ - while (len) {
- struct rte_mbuf *new_pkt = rsd->buf; - -
bufsz = min(get_buf_size(q->adapter, - rsd),
len); - new_pkt->data_len = bufsz; -
unmap_rx_buf(&rxq->fl); - len -= bufsz; -
npkt->next = new_pkt; - npkt = new_pkt; -
pkt->nb_segs++; - rsd = &rxq->fl.sdesc[rxq->fl.cidx]; +
if (len & F_RSPD_NEWBUF) { + if (q->offset > 0) { +
free_rx_bufs(&rxq->fl, 1); +
q->offset = 0; + } } -
npkt->next = NULL; - pkt->nb_segs--; + len =
G_RSPD_LEN(len); + ret = cxgbe_copy_rx_pkt(q, rxq, len, &pkt); +
if (unlikely(ret)) + break;
cxgbe_fill_mbuf_info(q->adapter, cpl, pkt); @@ -2379,6 +2449,7 @@ int
t4_sge_init(struct adapter *adap) sge_control = t4_read_reg(adap,
A_SGE_CONTROL); s->pktshift = G_PKTSHIFT(sge_control); s->stat_len =
(sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64; + s->fl_align =
cxgbe_fl_pkt_align(adap); ret = t4_sge_init_soft(adap); if (ret < 0) {
dev_err(adap, "%s: t4_sge_init_soft failed, error %d\n", @@ -2516,6
+2587,7 @@ int t4vf_sge_init(struct adapter *adap) s->stat_len =
((sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64); s->pktshift =
G_PKTSHIFT(sge_control); + s->fl_align = RTE_CACHE_LINE_SIZE; /* *
A FL with <= fl_starve_thres buffers is starving and a periodic -- 2.39.5
