Author: shurd Date: Sat Sep 23 16:46:30 2017 New Revision: 323954 URL: https://svnweb.freebsd.org/changeset/base/323954
Log: Have ifmp_ring_enqueue() abdicate instead of switch to a consumer Move TX out of the enqueue() path. As a result, we need to have ifmp_ring_check_drainage() pick up from the abdicate state. We also need to either enqueue the TX task, or check drainage after calling ifmp_ring_enqueue() to ensure it's sent. This change results in a 30% small packet forwarding improvement. Reviewed by: olivier, sbruno Approved by: sbruno (mentor) Sponsored by: Limelight Networks Differential Revision: https://reviews.freebsd.org/D12439 Modified: head/sys/net/iflib.c head/sys/net/mp_ring.c Modified: head/sys/net/iflib.c ============================================================================== --- head/sys/net/iflib.c Sat Sep 23 13:37:02 2017 (r323953) +++ head/sys/net/iflib.c Sat Sep 23 16:46:30 2017 (r323954) @@ -3515,8 +3515,7 @@ _task_fn_tx(void *context) } if (txq->ift_db_pending) ifmp_ring_enqueue(txq->ift_br, (void **)&txq, 1, TX_BATCH_SIZE); - else - ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); + ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); if (ctx->ifc_flags & IFC_LEGACY) IFDI_INTR_ENABLE(ctx); else { @@ -3718,16 +3717,14 @@ iflib_if_transmit(if_t ifp, struct mbuf *m) DBG_COUNTER_INC(tx_seen); err = ifmp_ring_enqueue(txq->ift_br, (void **)&m, 1, TX_BATCH_SIZE); + GROUPTASK_ENQUEUE(&txq->ift_task); if (err) { - GROUPTASK_ENQUEUE(&txq->ift_task); /* support forthcoming later */ #ifdef DRIVER_BACKPRESSURE txq->ift_closed = TRUE; #endif ifmp_ring_check_drainage(txq->ift_br, TX_BATCH_SIZE); m_freem(m); - } else if (TXQ_AVAIL(txq) < (txq->ift_size >> 1)) { - GROUPTASK_ENQUEUE(&txq->ift_task); } return (err); Modified: head/sys/net/mp_ring.c ============================================================================== --- head/sys/net/mp_ring.c Sat Sep 23 13:37:02 2017 (r323953) +++ head/sys/net/mp_ring.c Sat Sep 23 16:46:30 2017 (r323954) @@ -454,18 +454,12 @@ ifmp_ring_enqueue(struct ifmp_ring *r, void **items, i do { os.state = ns.state = r->state; ns.pidx_tail = pidx_stop; - ns.flags = BUSY; + if (os.flags == IDLE) + ns.flags = ABDICATED; } while (atomic_cmpset_rel_64(&r->state, os.state, ns.state) == 0); critical_exit(); counter_u64_add(r->enqueues, n); - /* - * Turn into a consumer if some other thread isn't active as a consumer - * already. - */ - if (os.flags != BUSY) - drain_ring_lockless(r, ns, os.flags, budget); - return (0); } #endif @@ -476,7 +470,9 @@ ifmp_ring_check_drainage(struct ifmp_ring *r, int budg union ring_state os, ns; os.state = r->state; - if (os.flags != STALLED || os.pidx_head != os.pidx_tail || r->can_drain(r) == 0) + if ((os.flags != STALLED && os.flags != ABDICATED) || // Only continue in STALLED and ABDICATED + os.pidx_head != os.pidx_tail || // Require work to be available + (os.flags != ABDICATED && r->can_drain(r) == 0)) // Can either drain, or everyone left return; MPASS(os.cidx != os.pidx_tail); /* implied by STALLED */ _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
