Module Name: src Committed By: ozaki-r Date: Mon Oct 26 01:44:48 UTC 2015
Modified Files: src/sys/dev/pci: if_vioif.c virtio.c virtioreg.h virtiovar.h Log Message: Support MSI-X in virtio Currently only vioif(4) uses the feature. knakahara@ helped to migrate to pci_intr_alloc(9). Thanks! To generate a diff of this commit: cvs rdiff -u -r1.16 -r1.17 src/sys/dev/pci/if_vioif.c cvs rdiff -u -r1.10 -r1.11 src/sys/dev/pci/virtio.c cvs rdiff -u -r1.3 -r1.4 src/sys/dev/pci/virtioreg.h cvs rdiff -u -r1.4 -r1.5 src/sys/dev/pci/virtiovar.h Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/dev/pci/if_vioif.c diff -u src/sys/dev/pci/if_vioif.c:1.16 src/sys/dev/pci/if_vioif.c:1.17 --- src/sys/dev/pci/if_vioif.c:1.16 Tue May 5 10:56:13 2015 +++ src/sys/dev/pci/if_vioif.c Mon Oct 26 01:44:48 2015 @@ -1,4 +1,4 @@ -/* $NetBSD: if_vioif.c,v 1.16 2015/05/05 10:56:13 ozaki-r Exp $ */ +/* $NetBSD: if_vioif.c,v 1.17 2015/10/26 01:44:48 ozaki-r Exp $ */ /* * Copyright (c) 2010 Minoura Makoto. @@ -26,7 +26,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.16 2015/05/05 10:56:13 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.17 2015/10/26 01:44:48 ozaki-r Exp $"); #ifdef _KERNEL_OPT #include "opt_net_mpsafe.h" @@ -487,6 +487,7 @@ vioif_attach(device_t parent, device_t s uint32_t features; struct ifnet *ifp = &sc->sc_ethercom.ec_if; u_int flags; + int r; if (vsc->sc_child != NULL) { aprint_normal(": child already attached for %s; " @@ -511,6 +512,7 @@ vioif_attach(device_t parent, device_t s #ifdef VIOIF_SOFTINT_INTR vsc->sc_flags |= VIRTIO_F_PCI_INTR_SOFTINT; #endif + vsc->sc_flags |= VIRTIO_F_PCI_INTR_MSIX; features = virtio_negotiate_features(vsc, (VIRTIO_NET_F_MAC | @@ -560,21 +562,6 @@ vioif_attach(device_t parent, device_t s aprint_normal(": Ethernet address %s\n", ether_sprintf(sc->sc_mac)); aprint_naive("\n"); - if (virtio_alloc_vq(vsc, &sc->sc_vq[0], 0, - MCLBYTES+sizeof(struct virtio_net_hdr), 2, - "rx") != 0) { - goto err; - } - vsc->sc_nvqs = 1; - sc->sc_vq[0].vq_done = vioif_rx_vq_done; - if (virtio_alloc_vq(vsc, &sc->sc_vq[1], 1, - (sizeof(struct virtio_net_hdr) - + (ETHER_MAX_LEN - ETHER_HDR_LEN)), - VIRTIO_NET_TX_MAXNSEGS + 1, - "tx") != 0) { - goto err; - } - #ifdef VIOIF_MPSAFE sc->sc_tx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); sc->sc_rx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); @@ -584,23 +571,51 @@ vioif_attach(device_t parent, device_t s #endif sc->sc_stopping = false; + /* + * Allocating a virtqueue for Rx + */ + r = virtio_alloc_vq(vsc, &sc->sc_vq[0], 0, + MCLBYTES+sizeof(struct virtio_net_hdr), 2, "rx"); + if (r != 0) + goto err; + vsc->sc_nvqs = 1; + sc->sc_vq[0].vq_done = vioif_rx_vq_done; + + /* + * Allocating a virtqueue for Tx + */ + r = virtio_alloc_vq(vsc, &sc->sc_vq[1], 1, + (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)), + VIRTIO_NET_TX_MAXNSEGS + 1, "tx"); + if (r != 0) + goto err; vsc->sc_nvqs = 2; sc->sc_vq[1].vq_done = vioif_tx_vq_done; + virtio_start_vq_intr(vsc, &sc->sc_vq[0]); virtio_stop_vq_intr(vsc, &sc->sc_vq[1]); /* not urgent; do it later */ - if ((features & VIRTIO_NET_F_CTRL_VQ) - && (features & VIRTIO_NET_F_CTRL_RX)) { - if (virtio_alloc_vq(vsc, &sc->sc_vq[2], 2, - NBPG, 1, "control") == 0) { - sc->sc_vq[2].vq_done = vioif_ctrl_vq_done; - cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); - mutex_init(&sc->sc_ctrl_wait_lock, - MUTEX_DEFAULT, IPL_NET); - sc->sc_ctrl_inuse = FREE; - virtio_start_vq_intr(vsc, &sc->sc_vq[2]); - vsc->sc_nvqs = 3; + + if ((features & VIRTIO_NET_F_CTRL_VQ) && + (features & VIRTIO_NET_F_CTRL_RX)) { + /* + * Allocating a virtqueue for control channel + */ + r = virtio_alloc_vq(vsc, &sc->sc_vq[2], 2, + NBPG, 1, "control"); + if (r != 0) { + aprint_error_dev(self, "failed to allocate " + "a virtqueue for control channel\n"); + goto skip; } + + sc->sc_vq[2].vq_done = vioif_ctrl_vq_done; + cv_init(&sc->sc_ctrl_wait, "ctrl_vq"); + mutex_init(&sc->sc_ctrl_wait_lock, MUTEX_DEFAULT, IPL_NET); + sc->sc_ctrl_inuse = FREE; + virtio_start_vq_intr(vsc, &sc->sc_vq[2]); + vsc->sc_nvqs = 3; } +skip: #ifdef VIOIF_MPSAFE flags = SOFTINT_NET | SOFTINT_MPSAFE; Index: src/sys/dev/pci/virtio.c diff -u src/sys/dev/pci/virtio.c:1.10 src/sys/dev/pci/virtio.c:1.11 --- src/sys/dev/pci/virtio.c:1.10 Thu Oct 15 02:40:38 2015 +++ src/sys/dev/pci/virtio.c Mon Oct 26 01:44:48 2015 @@ -1,4 +1,4 @@ -/* $NetBSD: virtio.c,v 1.10 2015/10/15 02:40:38 ozaki-r Exp $ */ +/* $NetBSD: virtio.c,v 1.11 2015/10/26 01:44:48 ozaki-r Exp $ */ /* * Copyright (c) 2010 Minoura Makoto. @@ -26,7 +26,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.10 2015/10/15 02:40:38 ozaki-r Exp $"); +__KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.11 2015/10/26 01:44:48 ozaki-r Exp $"); #include <sys/param.h> #include <sys/systm.h> @@ -49,6 +49,15 @@ static int virtio_match(device_t, cfdata static void virtio_attach(device_t, device_t, void *); static int virtio_detach(device_t, int); static int virtio_intr(void *arg); +static int virtio_msix_queue_intr(void *); +static int virtio_msix_config_intr(void *); +static int virtio_setup_msix_vectors(struct virtio_softc *); +static int virtio_setup_msix_interrupts(struct virtio_softc *, + struct pci_attach_args *); +static int virtio_setup_intx_interrupt(struct virtio_softc *, + struct pci_attach_args *); +static int virtio_setup_interrupts(struct virtio_softc *, + struct pci_attach_args *); static void virtio_soft_intr(void *arg); static void virtio_init_vq(struct virtio_softc *, struct virtqueue *, const bool); @@ -104,6 +113,203 @@ static const char *virtio_device_name[] }; #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) +#define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 +#define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 + +static int +virtio_setup_msix_vectors(struct virtio_softc *sc) +{ + int offset, vector, ret, qid; + + offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; + vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; + + bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); + ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); + aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", + vector, ret); + if (ret != vector) + return -1; + + for (qid = 0; qid < sc->sc_nvqs; qid++) { + offset = VIRTIO_CONFIG_QUEUE_SELECT; + bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); + + offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; + vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; + + bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); + ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); + aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", + vector, ret); + if (ret != vector) + return -1; + } + + return 0; +} + +static int +virtio_setup_msix_interrupts(struct virtio_softc *sc, + struct pci_attach_args *pa) +{ + device_t self = sc->sc_dev; + pci_chipset_tag_t pc = pa->pa_pc; + char intrbuf[PCI_INTRSTR_LEN]; + char const *intrstr; + int idx; + + idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) + pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); + + sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, + virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); + if (sc->sc_ihs[idx] == NULL) { + aprint_error_dev(self, "couldn't establish MSI-X for config\n"); + goto error; + } + + idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) + pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); + + sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, + virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); + if (sc->sc_ihs[idx] == NULL) { + aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); + goto error; + } + + if (virtio_setup_msix_vectors(sc) != 0) { + aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); + goto error; + } + + idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; + intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); + aprint_normal_dev(self, "config interrupting at %s\n", intrstr); + idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; + intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); + aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); + + return 0; + +error: + idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; + if (sc->sc_ihs[idx] != NULL) + pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); + idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; + if (sc->sc_ihs[idx] != NULL) + pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); + + return -1; +} + +static int +virtio_setup_intx_interrupt(struct virtio_softc *sc, struct pci_attach_args *pa) +{ + device_t self = sc->sc_dev; + pci_chipset_tag_t pc = pa->pa_pc; + char intrbuf[PCI_INTRSTR_LEN]; + char const *intrstr; + + if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) + pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); + + sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], + IPL_NET, virtio_intr, sc, device_xname(sc->sc_dev)); + if (sc->sc_ihs[0] == NULL) { + aprint_error_dev(self, "couldn't establish INTx\n"); + return -1; + } + + intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); + aprint_normal_dev(self, "interrupting at %s\n", intrstr); + + return 0; +} + +static int +virtio_setup_interrupts(struct virtio_softc *sc, struct pci_attach_args *pa) +{ + device_t self = sc->sc_dev; + pci_chipset_tag_t pc = pa->pa_pc; + int error; + int nmsix; + int counts[PCI_INTR_TYPE_SIZE]; + pci_intr_type_t max_type; + + nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); + aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); + + /* We need at least two: one for config and the other for queues */ + if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { + /* Try INTx only */ + max_type = PCI_INTR_TYPE_INTX; + counts[PCI_INTR_TYPE_INTX] = 1; + } else { + /* Try MSI-X first and INTx second */ + max_type = PCI_INTR_TYPE_MSIX; + counts[PCI_INTR_TYPE_MSIX] = 2; + counts[PCI_INTR_TYPE_MSI] = 0; + counts[PCI_INTR_TYPE_INTX] = 1; + } + + retry: + error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); + if (error != 0) { + aprint_error_dev(self, "couldn't map interrupt\n"); + return -1; + } + + if (pci_intr_type(sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { + sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, + KM_SLEEP); + if (sc->sc_ihs == NULL) { + pci_intr_release(pc, sc->sc_ihp, 2); + + /* Retry INTx */ + max_type = PCI_INTR_TYPE_INTX; + counts[PCI_INTR_TYPE_INTX] = 1; + goto retry; + } + + error = virtio_setup_msix_interrupts(sc, pa); + if (error != 0) { + kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); + pci_intr_release(pc, sc->sc_ihp, 2); + + /* Retry INTx */ + max_type = PCI_INTR_TYPE_INTX; + counts[PCI_INTR_TYPE_INTX] = 1; + goto retry; + } + + sc->sc_ihs_num = 2; + sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; + } else if (pci_intr_type(sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { + sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, + KM_SLEEP); + if (sc->sc_ihs == NULL) { + pci_intr_release(pc, sc->sc_ihp, 1); + return -1; + } + + error = virtio_setup_intx_interrupt(sc, pa); + if (error != 0) { + kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); + pci_intr_release(pc, sc->sc_ihp, 1); + return -1; + } + + sc->sc_ihs_num = 1; + sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; + } + + return 0; +} + static void virtio_attach(device_t parent, device_t self, void *aux) { @@ -113,9 +319,7 @@ virtio_attach(device_t parent, device_t pcitag_t tag = pa->pa_tag; int revision; pcireg_t id; - char const *intrstr; - pci_intr_handle_t ih; - char intrbuf[PCI_INTRSTR_LEN]; + int r; revision = PCI_REVISION(pa->pa_class); if (revision != 0) { @@ -166,29 +370,12 @@ virtio_attach(device_t parent, device_t return; } - if (pci_intr_map(pa, &ih)) { - aprint_error_dev(self, "couldn't map interrupt\n"); - virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); - return; - } - - intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); - - if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) - pci_intr_setattr(pc, &ih, PCI_INTR_MPSAFE, true); - - sc->sc_ih = pci_intr_establish_xname(pc, ih, sc->sc_ipl, virtio_intr, sc, - device_xname(sc->sc_dev)); - - if (sc->sc_ih == NULL) { - aprint_error_dev(self, "couldn't establish interrupt"); - if (intrstr != NULL) - aprint_error(" at %s", intrstr); - aprint_error("\n"); + r = virtio_setup_interrupts(sc, pa); + if (r != 0) { + aprint_error_dev(self, "failed to setup interrupts\n"); virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); return; } - aprint_normal_dev(self, "interrupting at %s\n", intrstr); sc->sc_soft_ih = NULL; if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { @@ -211,6 +398,7 @@ virtio_detach(device_t self, int flags) { struct virtio_softc *sc = device_private(self); int r; + int i; if (sc->sc_child != 0 && sc->sc_child != (void*)1) { r = config_detach(sc->sc_child, flags); @@ -219,10 +407,14 @@ virtio_detach(device_t self, int flags) } KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); KASSERT(sc->sc_vqs == 0); - if (sc->sc_ih != NULL) { - pci_intr_disestablish(sc->sc_pc, sc->sc_ih); - sc->sc_ih = NULL; + for (i = 0; i < sc->sc_ihs_num; i++) { + if (sc->sc_ihs[i] == NULL) + continue; + pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); } + pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); + kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); + sc->sc_ihs_num = 0; if (sc->sc_iosize) bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); sc->sc_iosize = 0; @@ -278,6 +470,14 @@ virtio_reinit_start(struct virtio_softc (vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE)); } + + /* MSI-X should have more than one handles where INTx has just one */ + if (sc->sc_ihs_num > 1) { + if (virtio_setup_msix_vectors(sc) != 0) { + aprint_error_dev(sc->sc_dev, "couldn't setup MSI-X vectors\n"); + return; + } + } } void @@ -411,6 +611,32 @@ virtio_intr(void *arg) return r; } +static int +virtio_msix_queue_intr(void *arg) +{ + struct virtio_softc *sc = arg; + int r = 0; + + if (sc->sc_intrhand != NULL) { + if (sc->sc_soft_ih != NULL) + softint_schedule(sc->sc_soft_ih); + else + r |= (sc->sc_intrhand)(sc); + } + + return r; +} + +static int +virtio_msix_config_intr(void *arg) +{ + struct virtio_softc *sc = arg; + + /* TODO: handle events */ + aprint_debug_dev(sc->sc_dev, "%s\n", __func__); + return 1; +} + static void virtio_soft_intr(void *arg) { Index: src/sys/dev/pci/virtioreg.h diff -u src/sys/dev/pci/virtioreg.h:1.3 src/sys/dev/pci/virtioreg.h:1.4 --- src/sys/dev/pci/virtioreg.h:1.3 Mon May 4 14:08:57 2015 +++ src/sys/dev/pci/virtioreg.h Mon Oct 26 01:44:48 2015 @@ -1,4 +1,4 @@ -/* $NetBSD: virtioreg.h,v 1.3 2015/05/04 14:08:57 ozaki-r Exp $ */ +/* $NetBSD: virtioreg.h,v 1.4 2015/10/26 01:44:48 ozaki-r Exp $ */ /* * Copyright (c) 2010 Minoura Makoto. @@ -96,6 +96,9 @@ #define VIRTIO_CONFIG_CONFIG_VECTOR 20 /* 16bit, optional */ #define VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI 20 #define VIRTIO_CONFIG_DEVICE_CONFIG_MSI 24 +/* MSI/MSI-X */ +#define VIRTIO_CONFIG_MSI_CONFIG_VECTOR 20 +#define VIRTIO_CONFIG_MSI_QUEUE_VECTOR 22 /* Virtqueue */ /* This marks a buffer as continuing via the next field. */ Index: src/sys/dev/pci/virtiovar.h diff -u src/sys/dev/pci/virtiovar.h:1.4 src/sys/dev/pci/virtiovar.h:1.5 --- src/sys/dev/pci/virtiovar.h:1.4 Fri Dec 19 06:54:40 2014 +++ src/sys/dev/pci/virtiovar.h Mon Oct 26 01:44:48 2015 @@ -1,4 +1,4 @@ -/* $NetBSD: virtiovar.h,v 1.4 2014/12/19 06:54:40 ozaki-r Exp $ */ +/* $NetBSD: virtiovar.h,v 1.5 2015/10/26 01:44:48 ozaki-r Exp $ */ /* * Copyright (c) 2010 Minoura Makoto. @@ -125,7 +125,9 @@ struct virtio_softc { bus_dma_tag_t sc_dmat; int sc_ipl; /* set by child */ - void *sc_ih; + pci_intr_handle_t *sc_ihp; + void **sc_ihs; + int sc_ihs_num; void *sc_soft_ih; int sc_flags; /* set by child */ @@ -151,6 +153,7 @@ struct virtio_softc { #define VIRTIO_F_PCI_INTR_MPSAFE (1 << 0) #define VIRTIO_F_PCI_INTR_SOFTINT (1 << 1) +#define VIRTIO_F_PCI_INTR_MSIX (1 << 2) /* public interface */ uint32_t virtio_negotiate_features(struct virtio_softc*, uint32_t);