Module Name: src
Committed By: msaitoh
Date: Wed Feb 1 10:47:13 UTC 2017
Modified Files:
src/sys/dev/pci/ixgbe: ix_txrx.c ixgbe.c ixgbe.h ixgbe_netbsd.h ixv.c
Log Message:
TX multiqueue. If you want to disable it, enable IXGBE_LEGACY_TX
in ixgbe_netbsd.h
To generate a diff of this commit:
cvs rdiff -u -r1.17 -r1.18 src/sys/dev/pci/ixgbe/ix_txrx.c
cvs rdiff -u -r1.66 -r1.67 src/sys/dev/pci/ixgbe/ixgbe.c
cvs rdiff -u -r1.21 -r1.22 src/sys/dev/pci/ixgbe/ixgbe.h
cvs rdiff -u -r1.5 -r1.6 src/sys/dev/pci/ixgbe/ixgbe_netbsd.h
cvs rdiff -u -r1.34 -r1.35 src/sys/dev/pci/ixgbe/ixv.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/pci/ixgbe/ix_txrx.c
diff -u src/sys/dev/pci/ixgbe/ix_txrx.c:1.17 src/sys/dev/pci/ixgbe/ix_txrx.c:1.18
--- src/sys/dev/pci/ixgbe/ix_txrx.c:1.17 Mon Jan 30 05:02:43 2017
+++ src/sys/dev/pci/ixgbe/ix_txrx.c Wed Feb 1 10:47:13 2017
@@ -59,7 +59,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
/*$FreeBSD: head/sys/dev/ixgbe/ix_txrx.c 301538 2016-06-07 04:51:50Z sephe $*/
-/*$NetBSD: ix_txrx.c,v 1.17 2017/01/30 05:02:43 msaitoh Exp $*/
+/*$NetBSD: ix_txrx.c,v 1.18 2017/02/01 10:47:13 msaitoh Exp $*/
#include "opt_inet.h"
#include "opt_inet6.h"
@@ -126,7 +126,6 @@ static __inline void ixgbe_rx_input(stru
static void ixgbe_setup_hw_rsc(struct rx_ring *);
-#ifdef IXGBE_LEGACY_TX
/*********************************************************************
* Transmit entry point
*
@@ -204,7 +203,7 @@ ixgbe_start(struct ifnet *ifp)
return;
}
-#else /* ! IXGBE_LEGACY_TX */
+#ifndef IXGBE_LEGACY_TX
/*
** Multiqueue Transmit Entry Point
@@ -214,7 +213,6 @@ int
ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m)
{
struct adapter *adapter = ifp->if_softc;
- struct ix_queue *que;
struct tx_ring *txr;
int i, err = 0;
#ifdef RSS
@@ -228,6 +226,7 @@ ixgbe_mq_start(struct ifnet *ifp, struct
* If everything is setup correctly, it should be the
* same bucket that the current CPU we're on is.
*/
+#if 0
#if __FreeBSD_version < 1100054
if (m->m_flags & M_FLOWID) {
#else
@@ -244,26 +243,29 @@ ixgbe_mq_start(struct ifnet *ifp, struct
"(%d)\n", bucket_id, adapter->num_queues);
#endif
} else
-#endif
+#endif /* RSS */
i = m->m_pkthdr.flowid % adapter->num_queues;
} else
- i = curcpu % adapter->num_queues;
+#endif
+ i = cpu_index(curcpu()) % adapter->num_queues;
/* Check for a hung queue and pick alternative */
if (((1 << i) & adapter->active_queues) == 0)
- i = ffsl(adapter->active_queues);
+ i = ffs64(adapter->active_queues);
txr = &adapter->tx_rings[i];
- que = &adapter->queues[i];
- err = drbr_enqueue(ifp, txr->br, m);
- if (err)
+ err = pcq_put(txr->txr_interq, m);
+ if (err == false) {
+ m_freem(m);
+ txr->pcq_drops.ev_count++;
return (err);
+ }
if (IXGBE_TX_TRYLOCK(txr)) {
ixgbe_mq_start_locked(ifp, txr);
IXGBE_TX_UNLOCK(txr);
} else
- softint_schedule(txr->txq_si);
+ softint_schedule(txr->txr_si);
return (0);
}
@@ -280,26 +282,12 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
return (ENETDOWN);
/* Process the queue */
-#if __FreeBSD_version < 901504
- next = drbr_dequeue(ifp, txr->br);
- while (next != NULL) {
- if ((err = ixgbe_xmit(txr, &next)) != 0) {
- if (next != NULL)
- err = drbr_enqueue(ifp, txr->br, next);
-#else
- while ((next = drbr_peek(ifp, txr->br)) != NULL) {
- if ((err = ixgbe_xmit(txr, &next)) != 0) {
- if (next == NULL) {
- drbr_advance(ifp, txr->br);
- } else {
- drbr_putback(ifp, txr->br, next);
- }
-#endif
+ while ((next = pcq_get(txr->txr_interq)) != NULL) {
+ if ((err = ixgbe_xmit(txr, next)) != 0) {
+ m_freem(next);
+ /* All errors are counted in ixgbe_xmit() */
break;
}
-#if __FreeBSD_version >= 901504
- drbr_advance(ifp, txr->br);
-#endif
enqueued++;
#if 0 // this is VF-only
#if __FreeBSD_version >= 1100036
@@ -311,14 +299,11 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
if (txr->tail < IXGBE_TDT(0) && next->m_flags & M_MCAST)
if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1);
#endif
-#endif
+#endif /* 0 */
/* Send a copy of the frame to the BPF listener */
bpf_mtap(ifp, next);
if ((ifp->if_flags & IFF_RUNNING) == 0)
break;
-#if __FreeBSD_version < 901504
- next = drbr_dequeue(ifp, txr->br);
-#endif
}
if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD)
@@ -331,36 +316,18 @@ ixgbe_mq_start_locked(struct ifnet *ifp,
* Called from a taskqueue to drain queued transmit packets.
*/
void
-ixgbe_deferred_mq_start(void *arg, int pending)
+ixgbe_deferred_mq_start(void *arg)
{
struct tx_ring *txr = arg;
struct adapter *adapter = txr->adapter;
struct ifnet *ifp = adapter->ifp;
IXGBE_TX_LOCK(txr);
- if (!drbr_empty(ifp, txr->br))
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
IXGBE_TX_UNLOCK(txr);
}
-/*
- * Flush all ring buffers
- */
-void
-ixgbe_qflush(struct ifnet *ifp)
-{
- struct adapter *adapter = ifp->if_softc;
- struct tx_ring *txr = adapter->tx_rings;
- struct mbuf *m;
-
- for (int i = 0; i < adapter->num_queues; i++, txr++) {
- IXGBE_TX_LOCK(txr);
- while ((m = buf_ring_dequeue_sc(txr->br)) != NULL)
- m_freem(m);
- IXGBE_TX_UNLOCK(txr);
- }
- if_qflush(ifp);
-}
#endif /* IXGBE_LEGACY_TX */
@@ -724,8 +691,13 @@ ixgbe_free_transmit_buffers(struct tx_ri
}
}
#ifndef IXGBE_LEGACY_TX
- if (txr->br != NULL)
- buf_ring_free(txr->br, M_DEVBUF);
+ if (txr->txr_interq != NULL) {
+ struct mbuf *m;
+
+ while ((m = pcq_get(txr->txr_interq)) != NULL)
+ m_freem(m);
+ pcq_destroy(txr->txr_interq);
+ }
#endif
if (txr->tx_buffers != NULL) {
free(txr->tx_buffers, M_DEVBUF);
@@ -2325,9 +2297,8 @@ ixgbe_allocate_queues(struct adapter *ad
}
#ifndef IXGBE_LEGACY_TX
/* Allocate a buf ring */
- txr->br = buf_ring_alloc(IXGBE_BR_SIZE, M_DEVBUF,
- M_WAITOK, &txr->tx_mtx);
- if (txr->br == NULL) {
+ txr->txr_interq = pcq_create(IXGBE_BR_SIZE, KM_SLEEP);
+ if (txr->txr_interq == NULL) {
aprint_error_dev(dev,
"Critical Failure setting up buf ring\n");
error = ENOMEM;
Index: src/sys/dev/pci/ixgbe/ixgbe.c
diff -u src/sys/dev/pci/ixgbe/ixgbe.c:1.66 src/sys/dev/pci/ixgbe/ixgbe.c:1.67
--- src/sys/dev/pci/ixgbe/ixgbe.c:1.66 Wed Jan 25 13:08:31 2017
+++ src/sys/dev/pci/ixgbe/ixgbe.c Wed Feb 1 10:47:13 2017
@@ -59,7 +59,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
/*$FreeBSD: head/sys/dev/ixgbe/if_ix.c 302384 2016-07-07 03:39:18Z sbruno $*/
-/*$NetBSD: ixgbe.c,v 1.66 2017/01/25 13:08:31 msaitoh Exp $*/
+/*$NetBSD: ixgbe.c,v 1.67 2017/02/01 10:47:13 msaitoh Exp $*/
#include "opt_inet.h"
#include "opt_inet6.h"
@@ -749,7 +749,7 @@ ixgbe_detach(device_t dev, int flags)
for (int i = 0; i < adapter->num_queues; i++, que++, txr++) {
#ifndef IXGBE_LEGACY_TX
- softint_disestablish(txr->txq_si);
+ softint_disestablish(txr->txr_si);
#endif
softint_disestablish(que->que_si);
}
@@ -1030,6 +1030,35 @@ ixgbe_ioctl(struct ifnet * ifp, u_long c
case SIOCSIFMTU:
IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
break;
+#ifdef __NetBSD__
+ case SIOCINITIFADDR:
+ IOCTL_DEBUGOUT("ioctl: SIOCINITIFADDR");
+ break;
+ case SIOCGIFFLAGS:
+ IOCTL_DEBUGOUT("ioctl: SIOCGIFFLAGS");
+ break;
+ case SIOCGIFAFLAG_IN:
+ IOCTL_DEBUGOUT("ioctl: SIOCGIFAFLAG_IN");
+ break;
+ case SIOCGIFADDR:
+ IOCTL_DEBUGOUT("ioctl: SIOCGIFADDR");
+ break;
+ case SIOCGIFMTU:
+ IOCTL_DEBUGOUT("ioctl: SIOCGIFMTU (Get Interface MTU)");
+ break;
+ case SIOCGIFCAP:
+ IOCTL_DEBUGOUT("ioctl: SIOCGIFCAP (Get IF cap)");
+ break;
+ case SIOCGETHERCAP:
+ IOCTL_DEBUGOUT("ioctl: SIOCGETHERCAP (Get ethercap)");
+ break;
+ case SIOCGLIFADDR:
+ IOCTL_DEBUGOUT("ioctl: SIOCGLIFADDR (Get Interface addr)");
+ break;
+ case SIOCAIFADDR:
+ IOCTL_DEBUGOUT("ioctl: SIOCAIFADDR (add/chg IF alias)");
+ break;
+#endif
default:
IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)", (int)command);
break;
@@ -1540,7 +1569,7 @@ ixgbe_handle_que(void *context)
IXGBE_TX_LOCK(txr);
ixgbe_txeof(txr);
#ifndef IXGBE_LEGACY_TX
- if (!drbr_empty(ifp, txr->br))
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
#else
if (!IFQ_IS_EMPTY(&ifp->if_snd))
@@ -1601,7 +1630,7 @@ ixgbe_legacy_irq(void *arg)
if (!IFQ_IS_EMPTY(&ifp->if_snd))
ixgbe_start_locked(txr, ifp);
#else
- if (!drbr_empty(ifp, txr->br))
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
#endif
IXGBE_TX_UNLOCK(txr);
@@ -1626,7 +1655,7 @@ ixgbe_legacy_irq(void *arg)
if (more)
#ifndef IXGBE_LEGACY_TX
- softint_schedule(txr->txq_si);
+ softint_schedule(txr->txr_si);
#else
softint_schedule(que->que_si);
#endif
@@ -1652,7 +1681,6 @@ ixgbe_msix_que(void *arg)
bool more;
u32 newitr = 0;
-
/* Protect against spurious interrupts */
if ((ifp->if_flags & IFF_RUNNING) == 0)
return 0;
@@ -1673,7 +1701,7 @@ ixgbe_msix_que(void *arg)
if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd))
ixgbe_start_locked(txr, ifp);
#else
- if (!drbr_empty(ifp, txr->br))
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
#endif
IXGBE_TX_UNLOCK(txr);
@@ -2562,7 +2590,7 @@ alloc_retry:
* processing contexts.
*/
#ifndef IXGBE_LEGACY_TX
- txr->txq_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start,
+ txr->txr_si = softint_establish(SOFTINT_NET, ixgbe_deferred_mq_start,
txr);
#endif
que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que, que);
@@ -2717,7 +2745,7 @@ ixgbe_allocate_msix(struct adapter *adap
}
aprint_normal("\n");
#ifndef IXGBE_LEGACY_TX
- txr->txq_si = softint_establish(SOFTINT_NET,
+ txr->txr_si = softint_establish(SOFTINT_NET,
ixgbe_deferred_mq_start, txr);
#endif
que->que_si = softint_establish(SOFTINT_NET, ixgbe_handle_que,
@@ -2969,18 +2997,22 @@ ixgbe_setup_interface(device_t dev, stru
#endif
#ifndef IXGBE_LEGACY_TX
ifp->if_transmit = ixgbe_mq_start;
- ifp->if_qflush = ixgbe_qflush;
#else
- ifp->if_start = ixgbe_start;
IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 2);
#if 0
ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 2;
#endif
IFQ_SET_READY(&ifp->if_snd);
#endif
+ ifp->if_start = ixgbe_start;
if_initialize(ifp);
ether_ifattach(ifp, adapter->hw.mac.addr);
+#ifndef IXGBE_LEGACY_TX
+#if 0 /* We use per TX queue softint */
+ if_deferred_start_init(ifp, ixgbe_deferred_mq_start);
+#endif
+#endif
if_register(ifp);
ether_set_ifflags_cb(ec, ixgbe_ifflags_cb);
@@ -4736,9 +4768,9 @@ ixgbe_add_hw_stats(struct adapter *adapt
NULL, adapter->queues[i].evnamebuf,
"Queue Packets Transmitted");
#ifndef IXGBE_LEGACY_TX
- evcnt_attach_dynamic(&txr->br->br_drops, EVCNT_TYPE_MISC,
+ evcnt_attach_dynamic(&txr->pcq_drops, EVCNT_TYPE_MISC,
NULL, adapter->queues[i].evnamebuf,
- "Packets dropped in buf_ring");
+ "Packets dropped in pcq");
#endif
#ifdef LRO
Index: src/sys/dev/pci/ixgbe/ixgbe.h
diff -u src/sys/dev/pci/ixgbe/ixgbe.h:1.21 src/sys/dev/pci/ixgbe/ixgbe.h:1.22
--- src/sys/dev/pci/ixgbe/ixgbe.h:1.21 Mon Jan 30 05:02:43 2017
+++ src/sys/dev/pci/ixgbe/ixgbe.h Wed Feb 1 10:47:13 2017
@@ -59,7 +59,7 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.h 303890 2016-08-09 19:32:06Z dumbbell $*/
-/*$NetBSD: ixgbe.h,v 1.21 2017/01/30 05:02:43 msaitoh Exp $*/
+/*$NetBSD: ixgbe.h,v 1.22 2017/02/01 10:47:13 msaitoh Exp $*/
#ifndef _IXGBE_H_
@@ -69,9 +69,7 @@
#include <sys/param.h>
#include <sys/reboot.h>
#include <sys/systm.h>
-#if __FreeBSD_version >= 800000
-#include <sys/buf_ring.h>
-#endif
+#include <sys/pcq.h>
#include <sys/mbuf.h>
#include <sys/protosw.h>
#include <sys/socket.h>
@@ -107,6 +105,7 @@
#include <sys/workqueue.h>
#include <sys/cpu.h>
#include <sys/interrupt.h>
+#include <sys/bitops.h>
#ifdef PCI_IOV
#include <sys/nv.h>
@@ -390,8 +389,8 @@ struct tx_ring {
ixgbe_dma_tag_t *txtag;
char mtx_name[16];
#ifndef IXGBE_LEGACY_TX
- struct buf_ring *br;
- void *txq_si;
+ pcq_t *txr_interq;
+ void *txr_si;
#endif
#ifdef IXGBE_FDIR
u16 atr_sample;
@@ -404,6 +403,7 @@ struct tx_ring {
struct evcnt no_tx_map_avail;
struct evcnt no_desc_avail;
struct evcnt total_packets;
+ struct evcnt pcq_drops;
};
@@ -757,15 +757,13 @@ ixv_check_ether_addr(u8 *addr)
/* Shared Prototypes */
-#ifdef IXGBE_LEGACY_TX
void ixgbe_start(struct ifnet *);
void ixgbe_start_locked(struct tx_ring *, struct ifnet *);
-#else /* ! IXGBE_LEGACY_TX */
+#ifndef IXGBE_LEGACY_TX
int ixgbe_mq_start(struct ifnet *, struct mbuf *);
int ixgbe_mq_start_locked(struct ifnet *, struct tx_ring *);
-void ixgbe_qflush(struct ifnet *);
-void ixgbe_deferred_mq_start(void *, int);
-#endif /* IXGBE_LEGACY_TX */
+void ixgbe_deferred_mq_start(void *);
+#endif /* !IXGBE_LEGACY_TX */
int ixgbe_allocate_queues(struct adapter *);
int ixgbe_allocate_transmit_buffers(struct tx_ring *);
Index: src/sys/dev/pci/ixgbe/ixgbe_netbsd.h
diff -u src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.5 src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.6
--- src/sys/dev/pci/ixgbe/ixgbe_netbsd.h:1.5 Thu Dec 1 06:56:28 2016
+++ src/sys/dev/pci/ixgbe/ixgbe_netbsd.h Wed Feb 1 10:47:13 2017
@@ -1,4 +1,4 @@
-/*$NetBSD: ixgbe_netbsd.h,v 1.5 2016/12/01 06:56:28 msaitoh Exp $*/
+/*$NetBSD: ixgbe_netbsd.h,v 1.6 2017/02/01 10:47:13 msaitoh Exp $*/
/*
* Copyright (c) 2011 The NetBSD Foundation, Inc.
* All rights reserved.
@@ -31,7 +31,9 @@
#ifndef _IXGBE_NETBSD_H
#define _IXGBE_NETBSD_H
+#if 1 /* Enable this if you don't want to use TX multiqueue function */
#define IXGBE_LEGACY_TX 1
+#endif
#define ETHERCAP_VLAN_HWFILTER 0
#define ETHERCAP_VLAN_HWCSUM 0
Index: src/sys/dev/pci/ixgbe/ixv.c
diff -u src/sys/dev/pci/ixgbe/ixv.c:1.34 src/sys/dev/pci/ixgbe/ixv.c:1.35
--- src/sys/dev/pci/ixgbe/ixv.c:1.34 Mon Jan 30 06:11:56 2017
+++ src/sys/dev/pci/ixgbe/ixv.c Wed Feb 1 10:47:13 2017
@@ -31,7 +31,7 @@
******************************************************************************/
/*$FreeBSD: head/sys/dev/ixgbe/if_ixv.c 302384 2016-07-07 03:39:18Z sbruno $*/
-/*$NetBSD: ixv.c,v 1.34 2017/01/30 06:11:56 msaitoh Exp $*/
+/*$NetBSD: ixv.c,v 1.35 2017/02/01 10:47:13 msaitoh Exp $*/
#include "opt_inet.h"
#include "opt_inet6.h"
@@ -527,7 +527,9 @@ ixv_detach(device_t dev, int flags)
for (int i = 0; i < adapter->num_queues; i++, que++) {
#ifndef IXGBE_LEGACY_TX
- softint_disestablish(txr->txq_si);
+ struct tx_ring *txr = adapter->tx_rings;
+
+ softint_disestablish(txr->txr_si);
#endif
softint_disestablish(que->que_si);
}
@@ -858,8 +860,8 @@ ixv_handle_que(void *context)
more = ixgbe_rxeof(que);
IXGBE_TX_LOCK(txr);
ixgbe_txeof(txr);
-#if __FreeBSD_version >= 800000
- if (!drbr_empty(ifp, txr->br))
+#ifndef IXGBE_LEGACY_TX
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
#else
if (!IFQ_IS_EMPTY(&ifp->if_snd))
@@ -915,7 +917,7 @@ ixv_msix_que(void *arg)
if (!IFQ_IS_EMPTY(&adapter->ifp->if_snd))
ixgbe_start_locked(txr, ifp);
#else
- if (!drbr_empty(adapter->ifp, txr->br))
+ if (pcq_peek(txr->txr_interq) != NULL)
ixgbe_mq_start_locked(ifp, txr);
#endif
IXGBE_TX_UNLOCK(txr);
@@ -1392,7 +1394,7 @@ ixv_allocate_msix(struct adapter *adapte
aprint_normal("\n");
#ifndef IXGBE_LEGACY_TX
- txr->txq_si = softint_establish(SOFTINT_NET,
+ txr->txr_si = softint_establish(SOFTINT_NET,
ixgbe_deferred_mq_start, txr);
#endif
que->que_si = softint_establish(SOFTINT_NET, ixv_handle_que,
@@ -1603,16 +1605,19 @@ ixv_setup_interface(device_t dev, struct
ifp->if_softc = adapter;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = ixv_ioctl;
-#if __FreeBSD_version >= 800000
+#ifndef IXGBE_LEGACY_TX
ifp->if_transmit = ixgbe_mq_start;
- ifp->if_qflush = ixgbe_qflush;
-#else
- ifp->if_start = ixgbe_start;
#endif
+ ifp->if_start = ixgbe_start;
ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2;
if_initialize(ifp);
ether_ifattach(ifp, adapter->hw.mac.addr);
+#ifndef IXGBE_LEGACY_TX
+#if 0 /* We use per TX queue softint */
+ if_deferred_start_init(ifp, ixgbe_deferred_mq_start);
+#endif
+#endif
if_register(ifp);
ether_set_ifflags_cb(ec, ixv_ifflags_cb);
