This prepares vmx(4) for multi-queue operation, first by making use of msi-x
where available, and second by rearranging the queue structures to fit the
direction we're heading in. As with other drivers, here I'm reserving msi-x
vector 0 for events, then mapping tx/rx queues to the subsequent vectors.
Aside from the interrupt setup itself, the only change in behaviour here is
that queue setup is done after interrupt setup, as we'll need to know what type
of interrupt we're using to decide how many queues to use. This is how other
vmx drivers work so this must be safe.
I've tested this with esxi 6.7 and qemu. Can somone try this out on vmware
workstation or player please? I wouldn't expect those to be any different to
esxi in this respect, but it's always possible.
ok?
Index: if_vmx.c
===================================================================
RCS file: /cvs/src/sys/dev/pci/if_vmx.c,v
retrieving revision 1.55
diff -u -p -r1.55 if_vmx.c
--- if_vmx.c 27 Oct 2019 22:24:40 -0000 1.55
+++ if_vmx.c 25 May 2020 09:35:33 -0000
@@ -42,8 +42,7 @@
#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>
-#define NRXQUEUE 1
-#define NTXQUEUE 1
+#define VMX_MAX_QUEUES 1
#define NTXDESC 512 /* tx ring size */
#define NTXSEGS 8 /* tx descriptors per packet */
@@ -95,6 +94,7 @@ struct vmxnet3_txqueue {
struct vmxnet3_txring cmd_ring;
struct vmxnet3_comp_ring comp_ring;
struct vmxnet3_txq_shared *ts;
+ struct ifqueue *ifq;
};
struct vmxnet3_rxqueue {
@@ -103,6 +103,14 @@ struct vmxnet3_rxqueue {
struct vmxnet3_rxq_shared *rs;
};
+struct vmxnet3_queue {
+ struct vmxnet3_txqueue tx;
+ struct vmxnet3_rxqueue rx;
+ struct vmxnet3_softc *sc;
+ char intrname[8];
+ int intr;
+};
+
struct vmxnet3_softc {
struct device sc_dev;
struct arpcom sc_arpcom;
@@ -114,9 +122,11 @@ struct vmxnet3_softc {
bus_space_handle_t sc_ioh1;
bus_dma_tag_t sc_dmat;
void *sc_ih;
+ void *sc_qih[VMX_MAX_QUEUES];
+ int sc_nintr;
+ int sc_nqueues;
- struct vmxnet3_txqueue sc_txq[NTXQUEUE];
- struct vmxnet3_rxqueue sc_rxq[NRXQUEUE];
+ struct vmxnet3_queue sc_q[VMX_MAX_QUEUES];
struct vmxnet3_driver_shared *sc_ds;
u_int8_t *sc_mcast;
};
@@ -153,8 +163,8 @@ struct {
int vmxnet3_match(struct device *, void *, void *);
void vmxnet3_attach(struct device *, struct device *, void *);
int vmxnet3_dma_init(struct vmxnet3_softc *);
-int vmxnet3_alloc_txring(struct vmxnet3_softc *, int);
-int vmxnet3_alloc_rxring(struct vmxnet3_softc *, int);
+int vmxnet3_alloc_txring(struct vmxnet3_softc *, int, int);
+int vmxnet3_alloc_rxring(struct vmxnet3_softc *, int, int);
void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
@@ -164,6 +174,8 @@ void vmxnet3_enable_all_intrs(struct vmx
void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
int vmxnet3_intr(void *);
int vmxnet3_intr_intx(void *);
+int vmxnet3_intr_event(void *);
+int vmxnet3_intr_queue(void *);
void vmxnet3_evintr(struct vmxnet3_softc *);
void vmxnet3_txintr(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
void vmxnet3_rxintr(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
@@ -212,6 +224,7 @@ vmxnet3_attach(struct device *parent, st
u_int memtype, ver, macl, mach, intrcfg;
u_char enaddr[ETHER_ADDR_LEN];
int (*isr)(void *);
+ int i;
memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x10);
if (pci_mapreg_map(pa, 0x10, memtype, 0, &sc->sc_iot0, &sc->sc_ioh0,
@@ -241,18 +254,22 @@ vmxnet3_attach(struct device *parent, st
WRITE_BAR1(sc, VMXNET3_BAR1_UVRS, 1);
sc->sc_dmat = pa->pa_dmat;
- if (vmxnet3_dma_init(sc)) {
- printf(": failed to setup DMA\n");
- return;
- }
WRITE_CMD(sc, VMXNET3_CMD_GET_INTRCFG);
intrcfg = READ_BAR1(sc, VMXNET3_BAR1_CMD);
isr = vmxnet3_intr;
+ sc->sc_nintr = 0;
+ sc->sc_nqueues = 1;
switch (intrcfg & VMXNET3_INTRCFG_TYPE_MASK) {
case VMXNET3_INTRCFG_TYPE_AUTO:
case VMXNET3_INTRCFG_TYPE_MSIX:
+ if (pci_intr_map_msix(pa, 0, &ih) == 0) {
+ isr = vmxnet3_intr_event;
+ sc->sc_nintr = sc->sc_nqueues + 1;
+ break;
+ }
+
/* FALLTHROUGH */
case VMXNET3_INTRCFG_TYPE_MSI:
if (pci_intr_map_msi(pa, &ih) == 0)
@@ -273,6 +290,35 @@ vmxnet3_attach(struct device *parent, st
if (intrstr)
printf(": %s", intrstr);
+ if (sc->sc_nintr > 1) {
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ struct vmxnet3_queue *q;
+ int vec;
+
+ q = &sc->sc_q[i];
+ vec = i + 1;
+ if (pci_intr_map_msix(pa, vec, &ih) != 0) {
+ printf(", failed to map interrupt %d\n", vec);
+ return;
+ }
+ snprintf(q->intrname, sizeof(q->intrname), "%s:%d",
+ self->dv_xname, i);
+ sc->sc_qih[i] = pci_intr_establish(pa->pa_pc, ih,
+ IPL_NET | IPL_MPSAFE, vmxnet3_intr_queue, q,
+ q->intrname);
+
+ q->intr = vec;
+ q->sc = sc;
+ }
+ }
+
+ if (vmxnet3_dma_init(sc)) {
+ printf(": failed to setup DMA\n");
+ return;
+ }
+
+ printf(", %d queue%s", sc->sc_nqueues, sc->sc_nqueues > 1 ? "s" : "");
+
WRITE_CMD(sc, VMXNET3_CMD_GET_MACL);
macl = READ_BAR1(sc, VMXNET3_BAR1_CMD);
enaddr[0] = macl;
@@ -319,6 +365,13 @@ vmxnet3_attach(struct device *parent, st
if_attach(ifp);
ether_ifattach(ifp);
vmxnet3_link_state(sc);
+
+ if_attach_queues(ifp, sc->sc_nqueues);
+ if_attach_iqueues(ifp, sc->sc_nqueues);
+ for (i = 0; i < sc->sc_nqueues; i++) {
+ ifp->if_ifqs[i]->ifq_softc = &sc->sc_q[i].tx;
+ sc->sc_q[i].tx.ifq = ifp->if_ifqs[i];
+ }
}
int
@@ -328,25 +381,30 @@ vmxnet3_dma_init(struct vmxnet3_softc *s
struct vmxnet3_txq_shared *ts;
struct vmxnet3_rxq_shared *rs;
bus_addr_t ds_pa, qs_pa, mcast_pa;
- int i, queue, qs_len;
+ int i, queue, qs_len, intr;
u_int major, minor, release_code, rev;
- qs_len = NTXQUEUE * sizeof *ts + NRXQUEUE * sizeof *rs;
+ qs_len = sc->sc_nqueues * (sizeof *ts + sizeof *rs);
ts = vmxnet3_dma_allocmem(sc, qs_len, VMXNET3_DMADESC_ALIGN, &qs_pa);
if (ts == NULL)
return -1;
- for (queue = 0; queue < NTXQUEUE; queue++)
- sc->sc_txq[queue].ts = ts++;
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ sc->sc_q[queue].tx.ts = ts++;
rs = (void *)ts;
- for (queue = 0; queue < NRXQUEUE; queue++)
- sc->sc_rxq[queue].rs = rs++;
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ sc->sc_q[queue].rx.rs = rs++;
- for (queue = 0; queue < NTXQUEUE; queue++)
- if (vmxnet3_alloc_txring(sc, queue))
+ for (queue = 0; queue < sc->sc_nqueues; queue++) {
+ if (sc->sc_nintr > 0)
+ intr = queue + 1;
+ else
+ intr = 0;
+
+ if (vmxnet3_alloc_txring(sc, queue, intr))
return -1;
- for (queue = 0; queue < NRXQUEUE; queue++)
- if (vmxnet3_alloc_rxring(sc, queue))
+ if (vmxnet3_alloc_rxring(sc, queue, intr))
return -1;
+ }
sc->sc_mcast = vmxnet3_dma_allocmem(sc, 682 * ETHER_ADDR_LEN, 32,
&mcast_pa);
if (sc->sc_mcast == NULL)
@@ -387,14 +445,14 @@ vmxnet3_dma_init(struct vmxnet3_softc *s
ds->queue_shared = qs_pa;
ds->queue_shared_len = qs_len;
ds->mtu = VMXNET3_MAX_MTU;
- ds->ntxqueue = NTXQUEUE;
- ds->nrxqueue = NRXQUEUE;
+ ds->ntxqueue = sc->sc_nqueues;
+ ds->nrxqueue = sc->sc_nqueues;
ds->mcast_table = mcast_pa;
ds->automask = 1;
- ds->nintr = VMXNET3_NINTR;
+ ds->nintr = sc->sc_nintr;
ds->evintr = 0;
ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
+ for (i = 0; i < sc->sc_nintr; i++)
ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
WRITE_BAR1(sc, VMXNET3_BAR1_DSL, ds_pa);
WRITE_BAR1(sc, VMXNET3_BAR1_DSH, (u_int64_t)ds_pa >> 32);
@@ -402,9 +460,9 @@ vmxnet3_dma_init(struct vmxnet3_softc *s
}
int
-vmxnet3_alloc_txring(struct vmxnet3_softc *sc, int queue)
+vmxnet3_alloc_txring(struct vmxnet3_softc *sc, int queue, int intr)
{
- struct vmxnet3_txqueue *tq = &sc->sc_txq[queue];
+ struct vmxnet3_txqueue *tq = &sc->sc_q[queue].tx;
struct vmxnet3_txq_shared *ts;
struct vmxnet3_txring *ring = &tq->cmd_ring;
struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
@@ -435,16 +493,16 @@ vmxnet3_alloc_txring(struct vmxnet3_soft
ts->comp_ring_len = NTXCOMPDESC;
ts->driver_data = vtophys(tq);
ts->driver_data_len = sizeof *tq;
- ts->intr_idx = 0;
+ ts->intr_idx = intr;
ts->stopped = 1;
ts->error = 0;
return 0;
}
int
-vmxnet3_alloc_rxring(struct vmxnet3_softc *sc, int queue)
+vmxnet3_alloc_rxring(struct vmxnet3_softc *sc, int queue, int intr)
{
- struct vmxnet3_rxqueue *rq = &sc->sc_rxq[queue];
+ struct vmxnet3_rxqueue *rq = &sc->sc_q[queue].rx;
struct vmxnet3_rxq_shared *rs;
struct vmxnet3_rxring *ring;
struct vmxnet3_comp_ring *comp_ring;
@@ -487,7 +545,7 @@ vmxnet3_alloc_rxring(struct vmxnet3_soft
rs->comp_ring_len = NRXCOMPDESC;
rs->driver_data = vtophys(rq);
rs->driver_data_len = sizeof *rq;
- rs->intr_idx = 0;
+ rs->intr_idx = intr;
rs->stopped = 1;
rs->error = 0;
return 0;
@@ -677,7 +735,7 @@ vmxnet3_enable_all_intrs(struct vmxnet3_
int i;
sc->sc_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
+ for (i = 0; i < sc->sc_nintr; i++)
vmxnet3_enable_intr(sc, i);
}
@@ -687,7 +745,7 @@ vmxnet3_disable_all_intrs(struct vmxnet3
int i;
sc->sc_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
+ for (i = 0; i < sc->sc_nintr; i++)
vmxnet3_disable_intr(sc, i);
}
@@ -715,14 +773,41 @@ vmxnet3_intr(void *arg)
}
if (ifp->if_flags & IFF_RUNNING) {
- vmxnet3_rxintr(sc, &sc->sc_rxq[0]);
- vmxnet3_txintr(sc, &sc->sc_txq[0]);
+ vmxnet3_rxintr(sc, &sc->sc_q[0].rx);
+ vmxnet3_txintr(sc, &sc->sc_q[0].tx);
vmxnet3_enable_intr(sc, 0);
}
return 1;
}
+int
+vmxnet3_intr_event(void *arg)
+{
+ struct vmxnet3_softc *sc = arg;
+
+ if (sc->sc_ds->event) {
+ KERNEL_LOCK();
+ vmxnet3_evintr(sc);
+ KERNEL_UNLOCK();
+ }
+
+ vmxnet3_enable_intr(sc, 0);
+ return 1;
+}
+
+int
+vmxnet3_intr_queue(void *arg)
+{
+ struct vmxnet3_queue *q = arg;
+
+ vmxnet3_rxintr(q->sc, &q->rx);
+ vmxnet3_txintr(q->sc, &q->tx);
+ vmxnet3_enable_intr(q->sc, q->intr);
+
+ return 1;
+}
+
void
vmxnet3_evintr(struct vmxnet3_softc *sc)
{
@@ -742,10 +827,10 @@ vmxnet3_evintr(struct vmxnet3_softc *sc)
if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
WRITE_CMD(sc, VMXNET3_CMD_GET_STATUS);
- ts = sc->sc_txq[0].ts;
+ ts = sc->sc_q[0].tx.ts;
if (ts->stopped)
printf("%s: TX error 0x%x\n", ifp->if_xname, ts->error);
- rs = sc->sc_rxq[0].rs;
+ rs = sc->sc_q[0].rx.rs;
if (rs->stopped)
printf("%s: RX error 0x%x\n", ifp->if_xname, rs->error);
vmxnet3_init(sc);
@@ -761,7 +846,7 @@ vmxnet3_evintr(struct vmxnet3_softc *sc)
void
vmxnet3_txintr(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq)
{
- struct ifnet *ifp = &sc->sc_arpcom.ac_if;
+ struct ifqueue *ifq = tq->ifq;
struct vmxnet3_txring *ring = &tq->cmd_ring;
struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
struct vmxnet3_txcompdesc *txcd;
@@ -808,8 +893,8 @@ vmxnet3_txintr(struct vmxnet3_softc *sc,
comp_ring->gen = rgen;
ring->cons = cons;
- if (ifq_is_oactive(&ifp->if_snd))
- ifq_restart(&ifp->if_snd);
+ if (ifq_is_oactive(ifq))
+ ifq_restart(ifq);
}
void
@@ -842,7 +927,7 @@ vmxnet3_rxintr(struct vmxnet3_softc *sc,
idx = letoh32((rxcd->rxc_word0 >> VMXNET3_RXC_IDX_S) &
VMXNET3_RXC_IDX_M);
if (letoh32((rxcd->rxc_word0 >> VMXNET3_RXC_QID_S) &
- VMXNET3_RXC_QID_M) < NRXQUEUE)
+ VMXNET3_RXC_QID_M) < sc->sc_nqueues)
ring = &rq->cmd_ring[0];
else
ring = &rq->cmd_ring[1];
@@ -890,10 +975,10 @@ skip_buffer:
VMXNET3_RXC_QID_S) & VMXNET3_RXC_QID_M);
idx = (idx + 1) % NRXDESC;
- if (qid < NRXQUEUE) {
+ if (qid < sc->sc_nqueues) {
WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(qid), idx);
} else {
- qid -= NRXQUEUE;
+ qid -= sc->sc_nqueues;
WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(qid), idx);
}
}
@@ -1000,12 +1085,17 @@ vmxnet3_stop(struct ifnet *ifp)
WRITE_CMD(sc, VMXNET3_CMD_DISABLE);
- intr_barrier(sc->sc_ih);
+ if (sc->sc_nintr == 1)
+ intr_barrier(sc->sc_ih);
+ else {
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ intr_barrier(sc->sc_qih[queue]);
+ }
- for (queue = 0; queue < NTXQUEUE; queue++)
- vmxnet3_txstop(sc, &sc->sc_txq[queue]);
- for (queue = 0; queue < NRXQUEUE; queue++)
- vmxnet3_rxstop(sc, &sc->sc_rxq[queue]);
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ vmxnet3_txstop(sc, &sc->sc_q[queue].tx);
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ vmxnet3_rxstop(sc, &sc->sc_q[queue].rx);
}
void
@@ -1030,12 +1120,12 @@ vmxnet3_init(struct vmxnet3_softc *sc)
vmxnet3_reset(sc);
#endif
- for (queue = 0; queue < NTXQUEUE; queue++)
- vmxnet3_txinit(sc, &sc->sc_txq[queue]);
- for (queue = 0; queue < NRXQUEUE; queue++)
- vmxnet3_rxinit(sc, &sc->sc_rxq[queue]);
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ vmxnet3_txinit(sc, &sc->sc_q[queue].tx);
+ for (queue = 0; queue < sc->sc_nqueues; queue++)
+ vmxnet3_rxinit(sc, &sc->sc_q[queue].rx);
- for (queue = 0; queue < NRXQUEUE; queue++) {
+ for (queue = 0; queue < sc->sc_nqueues; queue++) {
WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(queue), 0);
WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(queue), 0);
}
@@ -1092,7 +1182,7 @@ vmxnet3_ioctl(struct ifnet *ifp, u_long
break;
case SIOCGIFRXR:
error = if_rxr_ioctl((struct if_rxrinfo *)ifr->ifr_data,
- NULL, JUMBO_LEN, &sc->sc_rxq[0].cmd_ring[0].rxr);
+ NULL, JUMBO_LEN, &sc->sc_q[0].rx.cmd_ring[0].rxr);
break;
default:
error = ether_ioctl(ifp, &sc->sc_arpcom, cmd, data);
@@ -1131,7 +1221,7 @@ vmxnet3_start(struct ifqueue *ifq)
{
struct ifnet *ifp = ifq->ifq_if;
struct vmxnet3_softc *sc = ifp->if_softc;
- struct vmxnet3_txqueue *tq = sc->sc_txq;
+ struct vmxnet3_txqueue *tq = ifq->ifq_softc;
struct vmxnet3_txring *ring = &tq->cmd_ring;
struct vmxnet3_txdesc *txd, *sop;
bus_dmamap_t map;
