In GQI Queue Page List (GQI_QPL) mode each Tx queue copies outgoing
frames into a fixed QPL bounce FIFO (for example 512 pages = 2 MiB, or
~236 8,832-byte cacheline-aligned jumbo frames) even when nb_tx_desc is
configured to 2048. Under sustained jumbo bursts three bugs cause
stalls, packet drops, and corrupted FIFO accounting:
1. Entry cleanup in gve_tx_burst_qpl() only ran when nb_free <
free_thresh (32) or fifo_avail == 0. With 2048 descriptors, 236 jumbo
frames leave nb_free = 1811 and non-zero remainder bytes in
fifo_avail, so entry cleanup never ran. Reclaim when fifo_avail drops
below half of fifo_size, and stop counting normal FIFO-full
backpressure as stats.errors.
2. When is_fifo_avail() failed mid-burst after queuing frames, the
driver polled gve_tx_clean() once before writing qtx_tail, so the
device had not begun draining the queued frames, and one
GVE_TX_MAX_FREE_SZ batch after small frames could stop short of a
jumbo slot. Flush qtx_tail first and loop gve_tx_clean() while
next_to_clean advances.
3. For TSO packets, the header FIFO slice was allocated before checking
payload FIFO space (leaking header FIFO space on mid-packet abort),
and a malformed TSO mbuf with pkt_len <= hlen jumped to end_of_tx
without consuming the mbuf, stalling the queue indefinitely. Verify
combined aligned header + payload space up front with a 32-bit length
check, and free and count malformed TSO mbufs while tracking nb_sent
separately from nb_tx.
Fixes: a46583cf43c8 ("net/gve: support Rx/Tx")
Cc: [email protected]
Signed-off-by: Ryan Wolfe <[email protected]>
---
drivers/net/gve/gve_tx.c | 67 ++++++++++++++++++++++++++++------------
1 file changed, 48 insertions(+), 19 deletions(-)
diff --git a/drivers/net/gve/gve_tx.c b/drivers/net/gve/gve_tx.c
index 14495e3851..bf778a86e6 100644
--- a/drivers/net/gve/gve_tx.c
+++ b/drivers/net/gve/gve_tx.c
@@ -201,7 +201,7 @@ gve_tx_fill_seg_desc(volatile union gve_tx_desc *desc,
uint64_t ol_flags,
}
static inline bool
-is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)
+is_fifo_avail(struct gve_tx_queue *txq, uint32_t len)
{
if (txq->fifo_avail < len)
return false;
@@ -211,6 +211,27 @@ is_fifo_avail(struct gve_tx_queue *txq, uint16_t len)
return false;
return true;
}
+
+static inline bool
+gve_tx_reclaim_fifo_qpl(struct gve_tx_queue *txq, uint32_t tx_tail,
+ uint32_t need)
+{
+ if (txq->tx_tail != tx_tail) {
+ rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
+ txq->tx_tail = tx_tail;
+ }
+
+ for (;;) {
+ uint32_t prev_ntc = txq->next_to_clean;
+
+ gve_tx_clean(txq);
+ if (is_fifo_avail(txq, need))
+ return true;
+ if (txq->next_to_clean == prev_ntc)
+ return false;
+ }
+}
+
static inline uint64_t
gve_tx_alloc_from_fifo(struct gve_tx_queue *txq, uint16_t tx_id, uint16_t len)
{
@@ -261,13 +282,15 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf
**tx_pkts, uint16_t nb_pkts)
uint16_t sw_id = txq->sw_tail;
uint64_t ol_flags, fifo_addr;
uint16_t nb_used, i;
+ uint16_t nb_sent = 0;
uint64_t bytes = 0;
uint16_t nb_tx = 0;
uint32_t hlen;
txr = txq->tx_desc_ring;
- if (txq->nb_free < txq->free_thresh || txq->fifo_avail == 0)
+ if (txq->nb_free < txq->free_thresh ||
+ txq->fifo_avail < (txq->fifo_size >> 1))
gve_tx_clean(txq);
if (txq->sw_nb_free < txq->free_thresh)
@@ -305,21 +328,25 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf
**tx_pkts, uint16_t nb_pkts)
(uint32_t)(tx_offload.l2_len + tx_offload.l3_len +
tx_offload.l4_len) :
tx_pkt->pkt_len;
- if (!is_fifo_avail(txq, hlen)) {
- gve_tx_clean(txq);
- if (!is_fifo_avail(txq, hlen))
- goto end_of_tx;
- }
- fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
-
- /* For TSO, check if there's enough fifo space for data first */
if (ol_flags & RTE_MBUF_F_TX_TCP_SEG) {
- if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen)) {
- gve_tx_clean(txq);
- if (!is_fifo_avail(txq, tx_pkt->pkt_len - hlen))
- goto end_of_tx;
+ uint32_t data_len, need;
+
+ if (unlikely(tx_pkt->pkt_len <= hlen)) {
+ rte_pktmbuf_free(tx_pkt);
+ txq->stats.errors++;
+ continue;
}
+ data_len = tx_pkt->pkt_len - hlen;
+ need = RTE_ALIGN(hlen, RTE_CACHE_LINE_SIZE) +
+ RTE_ALIGN(data_len, RTE_CACHE_LINE_SIZE);
+ if (!is_fifo_avail(txq, need) &&
+ !gve_tx_reclaim_fifo_qpl(txq, tx_tail, need))
+ goto end_of_tx;
+ } else if (!is_fifo_avail(txq, hlen) &&
+ !gve_tx_reclaim_fifo_qpl(txq, tx_tail, hlen)) {
+ goto end_of_tx;
}
+ fifo_addr = gve_tx_alloc_from_fifo(txq, tx_id, hlen);
qpl_write_addr = (void *)(size_t)(fifo_addr + txq->fifo_base);
mbuf_header_addr = rte_pktmbuf_read(tx_pkt, 0, hlen,
qpl_write_addr);
@@ -367,17 +394,19 @@ gve_tx_burst_qpl(void *tx_queue, struct rte_mbuf
**tx_pkts, uint16_t nb_pkts)
tx_tail += nb_used;
bytes += first->pkt_len;
+ nb_sent++;
}
end_of_tx:
- if (nb_tx) {
- rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
- txq->tx_tail = tx_tail;
+ if (nb_sent) {
+ if (txq->tx_tail != tx_tail) {
+ rte_write32(rte_cpu_to_be_32(tx_tail), txq->qtx_tail);
+ txq->tx_tail = tx_tail;
+ }
txq->sw_tail = sw_id;
- txq->stats.packets += nb_tx;
+ txq->stats.packets += nb_sent;
txq->stats.bytes += bytes;
- txq->stats.errors += nb_pkts - nb_tx;
}
return nb_tx;