Module Name: src
Committed By: msaitoh
Date: Sat Jun 13 15:47:58 UTC 2015
Modified Files:
src/sys/dev/pci: if_wm.c if_wmreg.h
Log Message:
Add MSI/MSI-X support written by Kengo Nakahara. Some old devices' support
is written by me. It's disabled by default. If you'd like to use, define
WM_MSI_MSIX.
Tested with:
8254[3405617] (INTx even if it has MSI CAP because of a errata)
8257[12], 82583 ICH8, ICH10, PCH2, PCH_LPT(I21[78]) (MSI)
8257[456], 82580, I35[04], I21[01] (MSI-X)
Not tested:
82542, 82573, 80003, ICH9, PCH,
To generate a diff of this commit:
cvs rdiff -u -r1.334 -r1.335 src/sys/dev/pci/if_wm.c
cvs rdiff -u -r1.77 -r1.78 src/sys/dev/pci/if_wmreg.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/dev/pci/if_wm.c
diff -u src/sys/dev/pci/if_wm.c:1.334 src/sys/dev/pci/if_wm.c:1.335
--- src/sys/dev/pci/if_wm.c:1.334 Fri Jun 12 04:40:28 2015
+++ src/sys/dev/pci/if_wm.c Sat Jun 13 15:47:58 2015
@@ -1,4 +1,4 @@
-/* $NetBSD: if_wm.c,v 1.334 2015/06/12 04:40:28 msaitoh Exp $ */
+/* $NetBSD: if_wm.c,v 1.335 2015/06/13 15:47:58 msaitoh Exp $ */
/*
* Copyright (c) 2001, 2002, 2003, 2004 Wasabi Systems, Inc.
@@ -81,7 +81,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_wm.c,v 1.334 2015/06/12 04:40:28 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_wm.c,v 1.335 2015/06/13 15:47:58 msaitoh Exp $");
#ifdef _KERNEL_OPT
#include "opt_net_mpsafe.h"
@@ -154,6 +154,31 @@ int wm_debug = WM_DEBUG_TX | WM_DEBUG_RX
#define WM_MPSAFE 1
#endif
+#ifdef __HAVE_PCI_MSI_MSIX
+#if 0 /* off by default */
+#define WM_MSI_MSIX 1
+#endif
+#endif
+
+/*
+ * This device driver divides interrupt to TX, RX and link state.
+ * Each MSI-X vector indexes are below.
+ */
+#define WM_NINTR 3
+#define WM_TX_INTR_INDEX 0
+#define WM_RX_INTR_INDEX 1
+#define WM_LINK_INTR_INDEX 2
+#define WM_MAX_NINTR WM_NINTR
+
+/*
+ * This device driver set affinity to each interrupts like below (round-robin).
+ * If the number CPUs is less than the number of interrupts, this driver usase
+ * the same CPU for multiple interrupts.
+ */
+#define WM_TX_INTR_CPUID 0
+#define WM_RX_INTR_CPUID 1
+#define WM_LINK_INTR_CPUID 2
+
/*
* Transmit descriptor list size. Due to errata, we can only have
* 256 hardware descriptors in the ring on < 82544, but we use 4096
@@ -295,7 +320,13 @@ struct wm_softc {
int sc_flowflags; /* 802.3x flow control flags */
int sc_align_tweak;
- void *sc_ih; /* interrupt cookie */
+ void *sc_ihs[WM_MAX_NINTR]; /*
+ * interrupt cookie.
+ * legacy and msi use sc_ihs[0].
+ */
+ pci_intr_handle_t *sc_intrs; /* legacy and msi use sc_intrs[0] */
+ int sc_nintrs; /* number of interrupts */
+
callout_t sc_tick_ch; /* tick callout */
bool sc_stopping;
@@ -593,13 +624,18 @@ static int wm_nq_tx_offload(struct wm_so
static void wm_nq_start(struct ifnet *);
static void wm_nq_start_locked(struct ifnet *);
/* Interrupt */
-static void wm_txintr(struct wm_softc *);
-static void wm_rxintr(struct wm_softc *);
+static int wm_txeof(struct wm_softc *);
+static void wm_rxeof(struct wm_softc *);
static void wm_linkintr_gmii(struct wm_softc *, uint32_t);
static void wm_linkintr_tbi(struct wm_softc *, uint32_t);
static void wm_linkintr_serdes(struct wm_softc *, uint32_t);
static void wm_linkintr(struct wm_softc *, uint32_t);
-static int wm_intr(void *);
+static int wm_intr_legacy(void *);
+#ifdef WM_MSI_MSIX
+static int wm_txintr_msix(void *);
+static int wm_rxintr_msix(void *);
+static int wm_linkintr_msix(void *);
+#endif
/*
* Media related.
@@ -1368,7 +1404,11 @@ wm_attach(device_t parent, device_t self
prop_dictionary_t dict;
struct ifnet *ifp = &sc->sc_ethercom.ec_if;
pci_chipset_tag_t pc = pa->pa_pc;
+#ifndef WM_MSI_MSIX
pci_intr_handle_t ih;
+#else
+ bool intr_established = false;
+#endif
const char *intrstr = NULL;
const char *eetype, *xname;
bus_space_tag_t memt;
@@ -1424,6 +1464,19 @@ wm_attach(device_t parent, device_t self
sc->sc_type = WM_T_82542_2_0;
}
+ /*
+ * Disable MSI for Errata:
+ * "Message Signaled Interrupt Feature May Corrupt Write Transactions"
+ *
+ * 82544: Errata 25
+ * 82540: Errata 6 (easy to reproduce device timeout)
+ * 82545: Errata 4 (easy to reproduce device timeout)
+ * 82546: Errata 26 (easy to reproduce device timeout)
+ * 82541: Errata 7 (easy to reproduce device timeout)
+ */
+ if (sc->sc_type <= WM_T_82541_2)
+ pa->pa_flags &= ~PCI_FLAGS_MSI_OKAY;
+
if ((sc->sc_type == WM_T_82575) || (sc->sc_type == WM_T_82576)
|| (sc->sc_type == WM_T_82580)
|| (sc->sc_type == WM_T_I350) || (sc->sc_type == WM_T_I354)
@@ -1517,6 +1570,7 @@ wm_attach(device_t parent, device_t self
return;
}
+#ifndef WM_MSI_MSIX
/*
* Map and establish our interrupt.
*/
@@ -1528,8 +1582,8 @@ wm_attach(device_t parent, device_t self
#ifdef WM_MPSAFE
pci_intr_setattr(pc, &ih, PCI_INTR_MPSAFE, true);
#endif
- sc->sc_ih = pci_intr_establish(pc, ih, IPL_NET, wm_intr, sc);
- if (sc->sc_ih == NULL) {
+ sc->sc_ihs[0] = pci_intr_establish(pc, ih, IPL_NET, wm_intr_legacy,sc);
+ if (sc->sc_ihs[0] == NULL) {
aprint_error_dev(sc->sc_dev, "unable to establish interrupt");
if (intrstr != NULL)
aprint_error(" at %s", intrstr);
@@ -1537,6 +1591,171 @@ wm_attach(device_t parent, device_t self
return;
}
aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
+ sc->sc_nintrs = 1;
+#else /* WM_MSI_MSIX */
+ if (pci_msix_alloc_exact(pa, &sc->sc_intrs, WM_NINTR) == 0) {
+ /* 1st, try to use MSI-X */
+ void *vih;
+ kcpuset_t *affinity;
+
+ kcpuset_create(&affinity, false);
+
+ /*
+ * for TX
+ */
+ intrstr = pci_intr_string(pc, sc->sc_intrs[WM_TX_INTR_INDEX],
+ intrbuf, sizeof(intrbuf));
+#ifdef WM_MPSAFE
+ pci_intr_setattr(pc, &sc->sc_intrs[WM_TX_INTR_INDEX],
+ PCI_INTR_MPSAFE, true);
+#endif
+ vih = pci_intr_establish(pc, sc->sc_intrs[WM_TX_INTR_INDEX],
+ IPL_NET, wm_txintr_msix, sc);
+ if (vih == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to establish MSI-X(for TX)%s%s\n",
+ intrstr ? " at " : "", intrstr ? intrstr : "");
+ pci_intr_release(sc->sc_pc, sc->sc_intrs,
+ WM_NINTR);
+ goto msi;
+ }
+ kcpuset_zero(affinity);
+ /* Round-robin affinity */
+ kcpuset_set(affinity, WM_TX_INTR_CPUID % ncpu);
+ error = pci_intr_distribute(vih, affinity, NULL);
+ if (error == 0) {
+ aprint_normal_dev(sc->sc_dev,
+ "for TX interrupting at %s affinity to %u\n",
+ intrstr, WM_TX_INTR_CPUID % ncpu);
+ } else {
+ aprint_normal_dev(sc->sc_dev,
+ "for TX interrupting at %s\n",
+ intrstr);
+ }
+ sc->sc_ihs[WM_TX_INTR_INDEX] = vih;
+
+ /*
+ * for RX
+ */
+ intrstr = pci_intr_string(pc, sc->sc_intrs[WM_RX_INTR_INDEX],
+ intrbuf, sizeof(intrbuf));
+#ifdef WM_MPSAFE
+ pci_intr_setattr(pc, &sc->sc_intrs[WM_RX_INTR_INDEX],
+ PCI_INTR_MPSAFE, true);
+#endif
+ vih = pci_intr_establish(pc, sc->sc_intrs[WM_RX_INTR_INDEX],
+ IPL_NET, wm_rxintr_msix, sc);
+ if (vih == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to establish MSI-X(for RX)%s%s\n",
+ intrstr ? " at " : "", intrstr ? intrstr : "");
+ pci_intr_release(sc->sc_pc, sc->sc_intrs,
+ WM_NINTR);
+ goto msi;
+ }
+ kcpuset_zero(affinity);
+ kcpuset_set(affinity, WM_RX_INTR_CPUID % ncpu);
+ error = pci_intr_distribute(vih, affinity, NULL);
+ if (error == 0) {
+ aprint_normal_dev(sc->sc_dev,
+ "for RX interrupting at %s affinity to %u\n",
+ intrstr, WM_RX_INTR_CPUID % ncpu);
+ } else {
+ aprint_normal_dev(sc->sc_dev,
+ "for RX interrupting at %s\n",
+ intrstr);
+ }
+ sc->sc_ihs[WM_RX_INTR_INDEX] = vih;
+
+ /*
+ * for link state changing
+ */
+ intrstr = pci_intr_string(pc, sc->sc_intrs[WM_LINK_INTR_INDEX],
+ intrbuf, sizeof(intrbuf));
+#ifdef WM_MPSAFE
+ pci_intr_setattr(pc, &sc->sc_intrs[WM_LINK_INTR_INDEX],
+ PCI_INTR_MPSAFE, true);
+#endif
+ vih = pci_intr_establish(pc, sc->sc_intrs[WM_LINK_INTR_INDEX],
+ IPL_NET, wm_linkintr_msix, sc);
+ if (vih == NULL) {
+ aprint_error_dev(sc->sc_dev,
+ "unable to establish MSI-X(for LINK)%s%s\n",
+ intrstr ? " at " : "", intrstr ? intrstr : "");
+ pci_intr_release(sc->sc_pc, sc->sc_intrs,
+ WM_NINTR);
+ goto msi;
+ }
+ kcpuset_zero(affinity);
+ kcpuset_set(affinity, WM_LINK_INTR_CPUID % ncpu);
+ error = pci_intr_distribute(vih, affinity, NULL);
+ if (error == 0) {
+ aprint_normal_dev(sc->sc_dev,
+ "for LINK interrupting at %s affinity to %u\n",
+ intrstr, WM_LINK_INTR_CPUID % ncpu);
+ } else {
+ aprint_normal_dev(sc->sc_dev,
+ "for LINK interrupting at %s\n",
+ intrstr);
+ }
+ sc->sc_ihs[WM_LINK_INTR_INDEX] = vih;
+
+ sc->sc_nintrs = WM_NINTR;
+ kcpuset_destroy(affinity);
+ intr_established = true;
+ }
+
+msi:
+ if ((intr_established == false)
+ && (pci_msi_alloc_exact(pa, &sc->sc_intrs, 1) == 0)) {
+ /* 2nd, try to use MSI */
+ intrstr = pci_intr_string(pc, sc->sc_intrs[0], intrbuf,
+ sizeof(intrbuf));
+#ifdef WM_MPSAFE
+ pci_intr_setattr(pc, &sc->sc_intrs[0], PCI_INTR_MPSAFE, true);
+#endif
+ sc->sc_ihs[0] = pci_intr_establish(pc, sc->sc_intrs[0],
+ IPL_NET, wm_intr_legacy, sc);
+ if (sc->sc_ihs[0] == NULL) {
+ aprint_error_dev(sc->sc_dev, "unable to establish MSI\n");
+ pci_intr_release(sc->sc_pc, sc->sc_intrs,
+ 1);
+ goto intx;
+ }
+ aprint_normal_dev(sc->sc_dev, "MSI at %s\n", intrstr);
+
+ sc->sc_nintrs = 1;
+ intr_established = true;
+ }
+
+intx:
+ if ((intr_established == false)
+ && (pci_intx_alloc(pa, &sc->sc_intrs) == 0)) {
+ /* Last, try to use INTx */
+ intrstr = pci_intr_string(pc, sc->sc_intrs[0], intrbuf,
+ sizeof(intrbuf));
+#ifdef WM_MPSAFE
+ pci_intr_setattr(pc, &sc->sc_intrs[0], PCI_INTR_MPSAFE, true);
+#endif
+ sc->sc_ihs[0] = pci_intr_establish(pc, sc->sc_intrs[0],
+ IPL_NET, wm_intr_legacy, sc);
+ if (sc->sc_ihs[0] == NULL) {
+ aprint_error_dev(sc->sc_dev, "unable to establish MSI\n");
+ pci_intr_release(sc->sc_pc, sc->sc_intrs, 1);
+ goto int_failed;
+ }
+ aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
+
+ sc->sc_nintrs = 1;
+ intr_established = true;
+ }
+
+int_failed:
+ if (intr_established == false) {
+ aprint_error_dev(sc->sc_dev, "failed to allocate interrput\n");
+ return;
+ }
+#endif /* WM_MSI_MSIX */
/*
* Check the function ID (unit number of the chip).
@@ -2573,10 +2792,15 @@ wm_detach(device_t self, int flags __unu
bus_dmamem_free(sc->sc_dmat, &sc->sc_cd_seg, sc->sc_cd_rseg);
/* Disestablish the interrupt handler */
- if (sc->sc_ih != NULL) {
- pci_intr_disestablish(sc->sc_pc, sc->sc_ih);
- sc->sc_ih = NULL;
+ for (i = 0; i < sc->sc_nintrs; i++) {
+ if (sc->sc_ihs[i] != NULL) {
+ pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
+ sc->sc_ihs[i] = NULL;
+ }
}
+#ifdef WM_MSI_MSIX
+ pci_intr_release(sc->sc_pc, sc->sc_intrs, sc->sc_nintrs);
+#endif /* WM_MSI_MSIX */
/* Unmap the registers */
if (sc->sc_ss) {
@@ -2636,7 +2860,7 @@ wm_watchdog(struct ifnet *ifp)
* before we report an error.
*/
WM_TX_LOCK(sc);
- wm_txintr(sc);
+ wm_txeof(sc);
WM_TX_UNLOCK(sc);
if (sc->sc_txfree != WM_NTXDESC(sc)) {
@@ -3622,6 +3846,14 @@ wm_reset(struct wm_softc *sc)
/* Clear interrupt */
CSR_WRITE(sc, WMREG_IMC, 0xffffffffU);
+ if (sc->sc_nintrs > 1) {
+ if (sc->sc_type != WM_T_82574) {
+ CSR_WRITE(sc, WMREG_EIMC, 0xffffffffU);
+ CSR_WRITE(sc, WMREG_EIAC, 0);
+ } else {
+ CSR_WRITE(sc, WMREG_EIAC_82574, 0);
+ }
+ }
/* Stop the transmit and receive processes. */
CSR_WRITE(sc, WMREG_RCTL, 0);
@@ -3860,6 +4092,13 @@ wm_reset(struct wm_softc *sc)
/* Clear any pending interrupt events. */
CSR_WRITE(sc, WMREG_IMC, 0xffffffffU);
reg = CSR_READ(sc, WMREG_ICR);
+ if (sc->sc_nintrs > 1) {
+ if (sc->sc_type != WM_T_82574) {
+ CSR_WRITE(sc, WMREG_EIMC, 0xffffffffU);
+ CSR_WRITE(sc, WMREG_EIAC, 0);
+ } else
+ CSR_WRITE(sc, WMREG_EIAC_82574, 0);
+ }
/* reload sc_ctrl */
sc->sc_ctrl = CSR_READ(sc, WMREG_CTRL);
@@ -4289,11 +4528,124 @@ wm_init_locked(struct ifnet *ifp)
reg |= RXCSUM_IPV6OFL | RXCSUM_TUOFL;
CSR_WRITE(sc, WMREG_RXCSUM, reg);
+ /* Set up MSI-X */
+ if (sc->sc_nintrs > 1) {
+ uint32_t ivar;
+
+ if (sc->sc_type == WM_T_82575) {
+ /* Interrupt control */
+ reg = CSR_READ(sc, WMREG_CTRL_EXT);
+ reg |= CTRL_EXT_PBA | CTRL_EXT_EIAME | CTRL_EXT_NSICR;
+ CSR_WRITE(sc, WMREG_CTRL_EXT, reg);
+
+ /* TX */
+ CSR_WRITE(sc, WMREG_MSIXBM(WM_TX_INTR_INDEX),
+ EITR_TX_QUEUE0);
+ /* RX */
+ CSR_WRITE(sc, WMREG_MSIXBM(WM_RX_INTR_INDEX),
+ EITR_RX_QUEUE0);
+ /* Link status */
+ CSR_WRITE(sc, WMREG_MSIXBM(WM_LINK_INTR_INDEX),
+ EITR_OTHER);
+ } else if (sc->sc_type == WM_T_82574) {
+ /* Interrupt control */
+ reg = CSR_READ(sc, WMREG_CTRL_EXT);
+ reg |= CTRL_EXT_PBA | CTRL_EXT_EIAME;
+ CSR_WRITE(sc, WMREG_CTRL_EXT, reg);
+
+ /* TX, RX and Link status */
+ ivar = __SHIFTIN((IVAR_VALID_82574 | WM_TX_INTR_INDEX),
+ IVAR_TX_MASK_Q_82574(0));
+ ivar |= __SHIFTIN((IVAR_VALID_82574 |WM_RX_INTR_INDEX),
+ IVAR_RX_MASK_Q_82574(0));
+ ivar |=__SHIFTIN((IVAR_VALID_82574|WM_LINK_INTR_INDEX),
+ IVAR_OTHER_MASK);
+ CSR_WRITE(sc, WMREG_IVAR, ivar | IVAR_INT_ON_ALL_WB);
+ } else {
+ /* Interrupt control */
+ CSR_WRITE(sc, WMREG_GPIE, GPIE_NSICR
+ | GPIE_MULTI_MSIX | GPIE_EIAME
+ | GPIE_PBA);
+
+ switch (sc->sc_type) {
+ case WM_T_82580:
+ case WM_T_I350:
+ case WM_T_I354:
+ case WM_T_I210:
+ case WM_T_I211:
+ /* TX */
+ ivar = CSR_READ(sc, WMREG_IVAR_Q(0));
+ ivar &= ~IVAR_TX_MASK_Q(0);
+ ivar |= __SHIFTIN(
+ (WM_TX_INTR_INDEX | IVAR_VALID),
+ IVAR_TX_MASK_Q(0));
+ CSR_WRITE(sc, WMREG_IVAR_Q(0), ivar);
+
+ /* RX */
+ ivar = CSR_READ(sc, WMREG_IVAR_Q(0));
+ ivar &= ~IVAR_RX_MASK_Q(0);
+ ivar |= __SHIFTIN(
+ (WM_RX_INTR_INDEX | IVAR_VALID),
+ IVAR_RX_MASK_Q(0));
+ CSR_WRITE(sc, WMREG_IVAR_Q(0), ivar);
+ break;
+ case WM_T_82576:
+ /* TX */
+ ivar = CSR_READ(sc, WMREG_IVAR_Q_82576(0));
+ ivar &= ~IVAR_TX_MASK_Q_82576(0);
+ ivar |= __SHIFTIN(
+ (WM_TX_INTR_INDEX | IVAR_VALID),
+ IVAR_TX_MASK_Q_82576(0));
+ CSR_WRITE(sc, WMREG_IVAR_Q_82576(0), ivar);
+
+ /* RX */
+ ivar = CSR_READ(sc, WMREG_IVAR_Q_82576(0));
+ ivar &= ~IVAR_RX_MASK_Q_82576(0);
+ ivar |= __SHIFTIN(
+ (WM_RX_INTR_INDEX | IVAR_VALID),
+ IVAR_RX_MASK_Q_82576(0));
+ CSR_WRITE(sc, WMREG_IVAR_Q_82576(0), ivar);
+ break;
+ default:
+ break;
+ }
+
+ /* Link status */
+ ivar = __SHIFTIN((WM_LINK_INTR_INDEX | IVAR_VALID),
+ IVAR_MISC_OTHER);
+ CSR_WRITE(sc, WMREG_IVAR_MISC, ivar);
+ }
+ }
+
/* Set up the interrupt registers. */
CSR_WRITE(sc, WMREG_IMC, 0xffffffffU);
sc->sc_icr = ICR_TXDW | ICR_LSC | ICR_RXSEQ | ICR_RXDMT0 |
ICR_RXO | ICR_RXT0;
- CSR_WRITE(sc, WMREG_IMS, sc->sc_icr);
+ if (sc->sc_nintrs > 1) {
+ uint32_t mask;
+ switch (sc->sc_type) {
+ case WM_T_82574:
+ CSR_WRITE(sc, WMREG_EIAC_82574,
+ WMREG_EIAC_82574_MSIX_MASK);
+ sc->sc_icr |= WMREG_EIAC_82574_MSIX_MASK;
+ CSR_WRITE(sc, WMREG_IMS, sc->sc_icr);
+ break;
+ default:
+ if (sc->sc_type == WM_T_82575)
+ mask = EITR_RX_QUEUE0 |EITR_TX_QUEUE0
+ | EITR_OTHER;
+ else
+ mask = (1 << WM_RX_INTR_INDEX)
+ | (1 << WM_TX_INTR_INDEX)
+ | (1 << WM_LINK_INTR_INDEX);
+ CSR_WRITE(sc, WMREG_EIAC, mask);
+ CSR_WRITE(sc, WMREG_EIAM, mask);
+ CSR_WRITE(sc, WMREG_EIMS, mask);
+ CSR_WRITE(sc, WMREG_IMS, ICR_LSC);
+ break;
+ }
+ } else
+ CSR_WRITE(sc, WMREG_IMS, sc->sc_icr);
if ((sc->sc_type == WM_T_ICH8) || (sc->sc_type == WM_T_ICH9)
|| (sc->sc_type == WM_T_ICH10) || (sc->sc_type == WM_T_PCH)
@@ -4496,11 +4848,18 @@ wm_stop_locked(struct ifnet *ifp, int di
/*
* Clear the interrupt mask to ensure the device cannot assert its
* interrupt line.
- * Clear sc->sc_icr to ensure wm_intr() makes no attempt to service
- * any currently pending or shared interrupt.
+ * Clear sc->sc_icr to ensure wm_intr_legacy() makes no attempt to
+ * service any currently pending or shared interrupt.
*/
CSR_WRITE(sc, WMREG_IMC, 0xffffffffU);
sc->sc_icr = 0;
+ if (sc->sc_nintrs > 1) {
+ if (sc->sc_type != WM_T_82574) {
+ CSR_WRITE(sc, WMREG_EIMC, 0xffffffffU);
+ CSR_WRITE(sc, WMREG_EIAC, 0);
+ } else
+ CSR_WRITE(sc, WMREG_EIAC_82574, 0);
+ }
/* Release any queued transmit buffers. */
for (i = 0; i < WM_TXQUEUELEN(sc); i++) {
@@ -4884,7 +5243,7 @@ wm_start_locked(struct ifnet *ifp)
/* Get a work queue entry. */
if (sc->sc_txsfree < WM_TXQUEUE_GC(sc)) {
- wm_txintr(sc);
+ wm_txeof(sc);
if (sc->sc_txsfree == 0) {
DPRINTF(WM_DEBUG_TX,
("%s: TX: no free job descriptors\n",
@@ -5396,7 +5755,7 @@ wm_nq_start_locked(struct ifnet *ifp)
/* Get a work queue entry. */
if (sc->sc_txsfree < WM_TXQUEUE_GC(sc)) {
- wm_txintr(sc);
+ wm_txeof(sc);
if (sc->sc_txsfree == 0) {
DPRINTF(WM_DEBUG_TX,
("%s: TX: no free job descriptors\n",
@@ -5533,7 +5892,7 @@ wm_nq_start_locked(struct ifnet *ifp)
sc->sc_txdescs[nexttx].wtx_fields.wtxu_vlan =
htole16(VLAN_TAG_VALUE(mtag) & 0xffff);
} else {
- sc->sc_txdescs[nexttx].wtx_fields.wtxu_vlan = 0;
+ sc->sc_txdescs[nexttx].wtx_fields.wtxu_vlan =0;
}
dcmdlen = 0;
} else {
@@ -5645,20 +6004,22 @@ wm_nq_start_locked(struct ifnet *ifp)
/* Interrupt */
/*
- * wm_txintr:
+ * wm_txeof:
*
* Helper; handle transmit interrupts.
*/
-static void
-wm_txintr(struct wm_softc *sc)
+static int
+wm_txeof(struct wm_softc *sc)
{
struct ifnet *ifp = &sc->sc_ethercom.ec_if;
struct wm_txsoft *txs;
- uint8_t status;
+ bool processed = false;
+ int count = 0;
int i;
+ uint8_t status;
if (sc->sc_stopping)
- return;
+ return 0;
ifp->if_flags &= ~IFF_OACTIVE;
@@ -5684,6 +6045,8 @@ wm_txintr(struct wm_softc *sc)
break;
}
+ processed = true;
+ count++;
DPRINTF(WM_DEBUG_TX,
("%s: TX: job %d done: descs %d..%d\n",
device_xname(sc->sc_dev), i, txs->txs_firstdesc,
@@ -5726,26 +6089,32 @@ wm_txintr(struct wm_softc *sc)
DPRINTF(WM_DEBUG_TX,
("%s: TX: txsdirty -> %d\n", device_xname(sc->sc_dev), i));
+ if (count != 0)
+ rnd_add_uint32(&sc->rnd_source, count);
+
/*
* If there are no more pending transmissions, cancel the watchdog
* timer.
*/
if (sc->sc_txsfree == WM_TXQUEUELEN(sc))
ifp->if_timer = 0;
+
+ return processed;
}
/*
- * wm_rxintr:
+ * wm_rxeof:
*
* Helper; handle receive interrupts.
*/
static void
-wm_rxintr(struct wm_softc *sc)
+wm_rxeof(struct wm_softc *sc)
{
struct ifnet *ifp = &sc->sc_ethercom.ec_if;
struct wm_rxsoft *rxs;
struct mbuf *m;
int i, len;
+ int count = 0;
uint8_t status, errors;
uint16_t vlantag;
@@ -5769,6 +6138,7 @@ wm_rxintr(struct wm_softc *sc)
break;
}
+ count++;
if (__predict_false(sc->sc_rxdiscard)) {
DPRINTF(WM_DEBUG_RX,
("%s: RX: discarding contents of descriptor %d\n",
@@ -5938,6 +6308,8 @@ wm_rxintr(struct wm_softc *sc)
/* Update the receive pointer. */
sc->sc_rxptr = i;
+ if (count != 0)
+ rnd_add_uint32(&sc->rnd_source, count);
DPRINTF(WM_DEBUG_RX,
("%s: RX: rxptr -> %d\n", device_xname(sc->sc_dev), i));
@@ -6173,23 +6545,26 @@ wm_linkintr(struct wm_softc *sc, uint32_
}
/*
- * wm_intr:
+ * wm_intr_legacy:
*
- * Interrupt service routine.
+ * Interrupt service routine for INTx and MSI.
*/
static int
-wm_intr(void *arg)
+wm_intr_legacy(void *arg)
{
struct wm_softc *sc = arg;
struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- uint32_t icr;
+ uint32_t icr, rndval = 0;
int handled = 0;
+ DPRINTF(WM_DEBUG_TX,
+ ("%s: INTx: got intr\n", device_xname(sc->sc_dev)));
while (1 /* CONSTCOND */) {
icr = CSR_READ(sc, WMREG_ICR);
if ((icr & sc->sc_icr) == 0)
break;
- rnd_add_uint32(&sc->rnd_source, icr);
+ if (rndval == 0)
+ rndval = icr;
WM_RX_LOCK(sc);
@@ -6209,7 +6584,7 @@ wm_intr(void *arg)
WM_EVCNT_INCR(&sc->sc_ev_rxintr);
}
#endif
- wm_rxintr(sc);
+ wm_rxeof(sc);
WM_RX_UNLOCK(sc);
WM_TX_LOCK(sc);
@@ -6222,7 +6597,7 @@ wm_intr(void *arg)
WM_EVCNT_INCR(&sc->sc_ev_txdw);
}
#endif
- wm_txintr(sc);
+ wm_txeof(sc);
if (icr & (ICR_LSC|ICR_RXSEQ)) {
WM_EVCNT_INCR(&sc->sc_ev_linkintr);
@@ -6239,6 +6614,57 @@ wm_intr(void *arg)
}
}
+ rnd_add_uint32(&sc->rnd_source, rndval);
+
+ if (handled) {
+ /* Try to get more packets going. */
+ ifp->if_start(ifp);
+ }
+
+ return handled;
+}
+
+#ifdef WM_MSI_MSIX
+/*
+ * wm_txintr_msix:
+ *
+ * Interrupt service routine for TX complete interrupt for MSI-X.
+ */
+static int
+wm_txintr_msix(void *arg)
+{
+ struct wm_softc *sc = arg;
+ struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ int handled = 0;
+
+ DPRINTF(WM_DEBUG_TX,
+ ("%s: TX: got Tx intr\n", device_xname(sc->sc_dev)));
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMC, ICR_TXQ0); /* 82574 only */
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMC, EITR_TX_QUEUE0);
+ else
+ CSR_WRITE(sc, WMREG_EIMC, 1 << WM_TX_INTR_INDEX);
+
+ WM_TX_LOCK(sc);
+
+ if (sc->sc_stopping)
+ goto out;
+
+ WM_EVCNT_INCR(&sc->sc_ev_txdw);
+ handled = wm_txeof(sc);
+
+out:
+ WM_TX_UNLOCK(sc);
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMS, ICR_TXQ0); /* 82574 only */
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMS, EITR_TX_QUEUE0);
+ else
+ CSR_WRITE(sc, WMREG_EIMS, 1 << WM_TX_INTR_INDEX);
+
if (handled) {
/* Try to get more packets going. */
ifp->if_start(ifp);
@@ -6248,6 +6674,87 @@ wm_intr(void *arg)
}
/*
+ * wm_rxintr_msix:
+ *
+ * Interrupt service routine for RX interrupt for MSI-X.
+ */
+static int
+wm_rxintr_msix(void *arg)
+{
+ struct wm_softc *sc = arg;
+
+ DPRINTF(WM_DEBUG_TX,
+ ("%s: RX: got Rx intr\n", device_xname(sc->sc_dev)));
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMC, ICR_RXQ0); /* 82574 only */
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMC, EITR_RX_QUEUE0);
+ else
+ CSR_WRITE(sc, WMREG_EIMC, 1 << WM_RX_INTR_INDEX);
+
+ WM_RX_LOCK(sc);
+
+ if (sc->sc_stopping)
+ goto out;
+
+ WM_EVCNT_INCR(&sc->sc_ev_rxintr);
+ wm_rxeof(sc);
+
+out:
+ WM_RX_UNLOCK(sc);
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMS, ICR_RXQ0);
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMS, EITR_RX_QUEUE0);
+ else
+ CSR_WRITE(sc, WMREG_EIMS, 1 << WM_RX_INTR_INDEX);
+
+ return 1;
+}
+
+/*
+ * wm_linkintr_msix:
+ *
+ * Interrupt service routine for link status change for MSI-X.
+ */
+static int
+wm_linkintr_msix(void *arg)
+{
+ struct wm_softc *sc = arg;
+
+ DPRINTF(WM_DEBUG_TX,
+ ("%s: LINK: got link intr\n", device_xname(sc->sc_dev)));
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMC, ICR_OTHER); /* 82574 only */
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMC, EITR_OTHER);
+ else
+ CSR_WRITE(sc, WMREG_EIMC, 1 << WM_LINK_INTR_INDEX);
+ WM_TX_LOCK(sc);
+ if (sc->sc_stopping)
+ goto out;
+
+ WM_EVCNT_INCR(&sc->sc_ev_linkintr);
+ wm_linkintr(sc, ICR_LSC);
+
+out:
+ WM_TX_UNLOCK(sc);
+
+ if (sc->sc_type == WM_T_82574)
+ CSR_WRITE(sc, WMREG_IMS, ICR_OTHER | ICR_LSC); /* 82574 only */
+ else if (sc->sc_type == WM_T_82575)
+ CSR_WRITE(sc, WMREG_EIMS, EITR_OTHER);
+ else
+ CSR_WRITE(sc, WMREG_EIMS, 1 << WM_LINK_INTR_INDEX);
+
+ return 1;
+}
+#endif /* WM_MSI_MSIX */
+
+/*
* Media related.
* GMII, SGMII, TBI (and SERDES)
*/
Index: src/sys/dev/pci/if_wmreg.h
diff -u src/sys/dev/pci/if_wmreg.h:1.77 src/sys/dev/pci/if_wmreg.h:1.78
--- src/sys/dev/pci/if_wmreg.h:1.77 Sat Jun 6 04:39:12 2015
+++ src/sys/dev/pci/if_wmreg.h Sat Jun 13 15:47:58 2015
@@ -1,4 +1,4 @@
-/* $NetBSD: if_wmreg.h,v 1.77 2015/06/06 04:39:12 msaitoh Exp $ */
+/* $NetBSD: if_wmreg.h,v 1.78 2015/06/13 15:47:58 msaitoh Exp $ */
/*
* Copyright (c) 2001 Wasabi Systems, Inc.
@@ -299,6 +299,7 @@ struct livengood_tcpip_ctxdesc {
#define EERD_DATA_SHIFT 16 /* Offset to data in EEPROM read/write registers */
#define WMREG_CTRL_EXT 0x0018 /* Extended Device Control Register */
+#define CTRL_EXT_NSICR __BIT(0) /* Non Interrupt clear on read */
#define CTRL_EXT_GPI_EN(x) (1U << (x)) /* gpin interrupt enable */
#define CTRL_EXT_SWDPINS_SHIFT 4
#define CTRL_EXT_SWDPINS_MASK 0x0d
@@ -326,8 +327,10 @@ struct livengood_tcpip_ctxdesc {
#define CTRL_EXT_LINK_MODE_TBI 0x00C00000
#define CTRL_EXT_LINK_MODE_PCIE_SERDES 0x00C00000
#define CTRL_EXT_PHYPDEN 0x00100000
+#define CTRL_EXT_EIAME __BIT(24) /* Extended Interrupt Auto Mask En */
#define CTRL_EXT_I2C_ENA 0x02000000 /* I2C enable */
#define CTRL_EXT_DRV_LOAD 0x10000000
+#define CTRL_EXT_PBA __BIT(31) /* PBA Support */
#define WMREG_MDIC 0x0020 /* MDI Control Register */
#define MDIC_DATA(x) ((x) & 0xffff)
@@ -430,6 +433,11 @@ struct livengood_tcpip_ctxdesc {
#define ICR_MDAC (1U << 9) /* MDIO access complete */
#define ICR_RXCFG (1U << 10) /* Receiving /C/ */
#define ICR_GPI(x) (1U << (x)) /* general purpose interrupts */
+#define ICR_RXQ0 __BIT(20) /* 82574: Rx queue 0 interrupt */
+#define ICR_RXQ1 __BIT(21) /* 82574: Rx queue 1 interrupt */
+#define ICR_TXQ0 __BIT(22) /* 82574: Tx queue 0 interrupt */
+#define ICR_TXQ1 __BIT(23) /* 82574: Tx queue 1 interrupt */
+#define ICR_OTHER __BIT(24) /* 82574: Other interrupt */
#define ICR_INT (1U << 31) /* device generated an interrupt */
#define WMREG_ITR 0x00c4 /* Interrupt Throttling Register */
@@ -439,12 +447,40 @@ struct livengood_tcpip_ctxdesc {
#define WMREG_ICS 0x00c8 /* Interrupt Cause Set Register */
/* See ICR bits. */
+#define WMREG_IVAR 0x00e4 /* Interrupt Vector Allocation Register */
+#define WMREG_IVAR0 0x01700 /* Interrupt Vector Allocation */
+#define IVAR_ALLOC_MASK __BITS(0, 6) /* Bit 5 and 6 are reserved */
+#define IVAR_VALID __BIT(7)
+/* IVAR definitions for 82580 and newer */
+#define WMREG_IVAR_Q(x) (WMREG_IVAR0 + ((x) % 2) * 4)
+#define IVAR_TX_MASK_Q(x) (0x000000ff << (((x) % 2) == 0 ? 8 : 24))
+#define IVAR_RX_MASK_Q(x) (0x000000ff << (((x) % 2) == 0 ? 0 : 16))
+/* IVAR definitions for 82576 */
+#define WMREG_IVAR_Q_82576(x) (WMREG_IVAR0 + ((x) & 0x7) * 4)
+#define IVAR_TX_MASK_Q_82576(x) (0x000000ff << (((x) / 8) == 0 ? 8 : 24))
+#define IVAR_RX_MASK_Q_82576(x) (0x000000ff << (((x) / 8) == 0 ? 0 : 16))
+/* IVAR definitions for 82574 */
+#define IVAR_ALLOC_MASK_82574 __BITS(0, 2)
+#define IVAR_VALID_82574 __BIT(3)
+#define IVAR_TX_MASK_Q_82574(x) (0x0000000f << ((x) == 0 ? 8 : 12))
+#define IVAR_RX_MASK_Q_82574(x) (0x0000000f << ((x) == 0 ? 0 : 4))
+#define IVAR_OTHER_MASK __BITS(16, 19)
+#define IVAR_INT_ON_ALL_WB __BIT(31)
+
+#define WMREG_IVAR_MISC 0x01740 /* IVAR for other causes */
+#define IVAR_MISC_TCPTIMER __BITS(0, 7)
+#define IVAR_MISC_OTHER __BITS(8, 15)
+
#define WMREG_IMS 0x00d0 /* Interrupt Mask Set Register */
/* See ICR bits. */
#define WMREG_IMC 0x00d8 /* Interrupt Mask Clear Register */
/* See ICR bits. */
+#define WMREG_EIAC_82574 0x00dc /* Interrupt Auto Clear Register */
+#define WMREG_EIAC_82574_MSIX_MASK (ICR_RXQ0 | ICR_RXQ1 \
+ | ICR_TXQ0 | ICR_TXQ1 | ICR_OTHER)
+
#define WMREG_RCTL 0x0100 /* Receive Control */
#define RCTL_EN (1U << 1) /* receiver enable */
#define RCTL_SBP (1U << 2) /* store bad packets */
@@ -716,6 +752,12 @@ struct livengood_tcpip_ctxdesc {
#define PBA_ECC_STAT_CLR 0x00000002 /* Clear ECC error counter */
#define PBA_ECC_INT_EN 0x00000004 /* Enable ICR bit 5 on ECC error */
+#define WMREG_GPIE 0x01514 /* General Purpose Interrupt Enable */
+#define GPIE_NSICR __BIT(0) /* Non Selective Interrupt Clear */
+#define GPIE_MULTI_MSIX __BIT(4) /* Multiple MSIX */
+#define GPIE_EIAME __BIT(30) /* Extended Interrupt Auto Mask Ena. */
+#define GPIE_PBA __BIT(31) /* PBA support */
+
#define WMREG_EICS 0x01520 /* Ext. Interrupt Cause Set - WO */
#define WMREG_EIMS 0x01524 /* Ext. Interrupt Mask Set/Read - RW */
#define WMREG_EIMC 0x01528 /* Ext. Interrupt Mask Clear - WO */
@@ -724,6 +766,8 @@ struct livengood_tcpip_ctxdesc {
#define WMREG_EICR 0x01580 /* Ext. Interrupt Cause Read - R/clr */
+#define WMREG_MSIXBM(x) (0x1600 + (x) * 4) /* MSI-X Allocation */
+
#define EITR_RX_QUEUE0 0x00000001 /* Rx Queue 0 Interrupt */
#define EITR_RX_QUEUE1 0x00000002 /* Rx Queue 1 Interrupt */
#define EITR_RX_QUEUE2 0x00000004 /* Rx Queue 2 Interrupt */
