Module Name: src Committed By: yamaguchi Date: Thu Mar 23 02:57:54 UTC 2023
Modified Files: src/sys/dev/pci: if_vioif.c Log Message: vioif(4): reorganize functions iThis change is move of function and rename, and this is no functional change. To generate a diff of this commit: cvs rdiff -u -r1.100 -r1.101 src/sys/dev/pci/if_vioif.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/dev/pci/if_vioif.c diff -u src/sys/dev/pci/if_vioif.c:1.100 src/sys/dev/pci/if_vioif.c:1.101 --- src/sys/dev/pci/if_vioif.c:1.100 Thu Mar 23 02:52:29 2023 +++ src/sys/dev/pci/if_vioif.c Thu Mar 23 02:57:54 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: if_vioif.c,v 1.100 2023/03/23 02:52:29 yamaguchi Exp $ */ +/* $NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $ */ /* * Copyright (c) 2020 The NetBSD Foundation, Inc. @@ -27,7 +27,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.100 2023/03/23 02:52:29 yamaguchi Exp $"); +__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $"); #ifdef _KERNEL_OPT #include "opt_net_mpsafe.h" @@ -330,7 +330,7 @@ struct vioif_softc { void *sc_dmamem; void *sc_kmem; - void *sc_ctl_softint; + void *sc_cfg_softint; struct workqueue *sc_txrx_workqueue; bool sc_txrx_workqueue_sysctl; @@ -360,1171 +360,1213 @@ static int vioif_finalize_teardown(devic static int vioif_init(struct ifnet *); static void vioif_stop(struct ifnet *, int); static void vioif_start(struct ifnet *); -static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *); static int vioif_transmit(struct ifnet *, struct mbuf *); -static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *); static int vioif_ioctl(struct ifnet *, u_long, void *); static void vioif_watchdog(struct ifnet *); +static int vioif_ifflags(struct vioif_softc *); static int vioif_ifflags_cb(struct ethercom *); /* tx & rx */ -static void vioif_net_sched_handle(struct vioif_softc *, - struct vioif_netqueue *); -static int vioif_net_load_mbuf(struct virtio_softc *, - struct vioif_net_map *, struct mbuf *, int); -static void vioif_net_unload_mbuf(struct virtio_softc *, - struct vioif_net_map *); -static int vioif_net_enqueue_tx(struct virtio_softc *, struct virtqueue *, - int, struct vioif_net_map *); -static int vioif_net_enqueue_rx(struct virtio_softc *, struct virtqueue *, - int, struct vioif_net_map *); -static struct mbuf * - vioif_net_dequeue_commit(struct virtio_softc *, - struct virtqueue *, int, struct vioif_net_map *, int); +static int vioif_netqueue_init(struct vioif_softc *, + struct virtio_softc *, size_t, u_int); +static void vioif_netqueue_teardown(struct vioif_softc *, + struct virtio_softc *, size_t); static void vioif_net_intr_enable(struct vioif_softc *, struct virtio_softc *); static void vioif_net_intr_disable(struct vioif_softc *, struct virtio_softc *); +static void vioif_net_sched_handle(struct vioif_softc *, + struct vioif_netqueue *); /* rx */ static void vioif_populate_rx_mbufs_locked(struct vioif_softc *, struct vioif_netqueue *); -static void vioif_rx_queue_clear(struct vioif_softc *, struct virtio_softc *, - struct vioif_netqueue *); -static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *, - struct vioif_netqueue *, u_int, size_t *); static int vioif_rx_intr(void *); static void vioif_rx_handle(void *); +static void vioif_rx_queue_clear(struct vioif_softc *, + struct virtio_softc *, struct vioif_netqueue *); /* tx */ +static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *); +static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *); +static void vioif_deferred_transmit(void *); static int vioif_tx_intr(void *); static void vioif_tx_handle(void *); static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *, struct vioif_netqueue *); -static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *, - struct vioif_netqueue *, u_int); -static void vioif_deferred_transmit(void *); - -/* workqueue */ -static struct workqueue* - vioif_workq_create(const char *, pri_t, int, int); -static void vioif_workq_destroy(struct workqueue *); -static void vioif_workq_work(struct work *, void *); -static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); -static void vioif_work_add(struct workqueue *, struct vioif_work *); -static void vioif_work_wait(struct workqueue *, struct vioif_work *); -/* other control */ -static int vioif_get_link_status(struct vioif_softc *); -static void vioif_update_link_status(struct vioif_softc *); +/* controls */ +static int vioif_ctrl_intr(void *); static int vioif_ctrl_rx(struct vioif_softc *, int, bool); static int vioif_set_promisc(struct vioif_softc *, bool); static int vioif_set_allmulti(struct vioif_softc *, bool); static int vioif_set_rx_filter(struct vioif_softc *); static int vioif_rx_filter(struct vioif_softc *); static int vioif_set_mac_addr(struct vioif_softc *); -static int vioif_ctrl_intr(void *); -static int vioif_config_change(struct virtio_softc *); -static void vioif_ctl_softint(void *); static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int); + +/* config interrupt */ +static int vioif_config_change(struct virtio_softc *); +static void vioif_cfg_softint(void *); +static void vioif_update_link_status(struct vioif_softc *); + +/* others */ +static void vioif_alloc_queues(struct vioif_softc *); +static void vioif_free_queues(struct vioif_softc *); +static int vioif_alloc_mems(struct vioif_softc *); +static struct workqueue* + vioif_workq_create(const char *, pri_t, int, int); +static void vioif_workq_destroy(struct workqueue *); +static void vioif_work_set(struct vioif_work *, void(*)(void *), void *); +static void vioif_work_add(struct workqueue *, struct vioif_work *); +static void vioif_work_wait(struct workqueue *, struct vioif_work *); static int vioif_setup_sysctl(struct vioif_softc *); static void vioif_setup_stats(struct vioif_softc *); -static int vioif_ifflags(struct vioif_softc *); -static void vioif_intr_barrier(void); -static void vioif_notify(struct virtio_softc *, struct virtqueue *); CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc), vioif_match, vioif_attach, NULL, NULL); -static int -vioif_match(device_t parent, cfdata_t match, void *aux) -{ - struct virtio_attach_args *va = aux; - - if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK) - return 1; - - return 0; -} - -static int -vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, - bus_size_t size, int nsegs, const char *usage) +static void +vioif_intr_barrier(void) { - int r; - - r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, - nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); - - if (r != 0) { - aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " - "error code %d\n", usage, r); - } - return r; + /* wait for finish all interrupt handler */ + xc_barrier(0); } static void -vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) +vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq) { - if (*map) { - bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); - *map = NULL; - } + virtio_enqueue_commit(vsc, vq, -1, true); } static int -vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, - void *buf, bus_size_t size, int nsegs, int rw, const char *usage) +vioif_match(device_t parent, cfdata_t match, void *aux) { - int r; + struct virtio_attach_args *va = aux; - r = vioif_dmamap_create(sc, map, size, nsegs, usage); - if (r != 0) + if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK) return 1; - r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, - size, NULL, rw | BUS_DMA_NOWAIT); - if (r != 0) { - vioif_dmamap_destroy(sc, map); - aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " - "error code %d\n", usage, r); - } - - return r; + return 0; } -static void * -vioif_assign_mem(intptr_t *p, size_t size) +static void +vioif_attach(device_t parent, device_t self, void *aux) { - intptr_t rv; + struct vioif_softc *sc = device_private(self); + struct virtio_softc *vsc = device_private(parent); + struct vioif_netqueue *txq0; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + uint64_t features, req_features; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + u_int softint_flags; + int r, i, req_flags; + char xnamebuf[MAXCOMLEN]; + size_t netq_num; - rv = *p; - *p += size; + if (virtio_child(vsc) != NULL) { + aprint_normal(": child already attached for %s; " + "something wrong...\n", device_xname(parent)); + return; + } - return (void *)rv; -} + sc->sc_dev = self; + sc->sc_virtio = vsc; + sc->sc_link_state = LINK_STATE_UNKNOWN; -static void -vioif_alloc_queues(struct vioif_softc *sc) -{ - int nvq_pairs = sc->sc_max_nvq_pairs; - size_t nvqs, netq_num; + sc->sc_max_nvq_pairs = 1; + sc->sc_req_nvq_pairs = 1; + sc->sc_act_nvq_pairs = 1; + sc->sc_txrx_workqueue_sysctl = true; + sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; + sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; + sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; + sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; - KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); + mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); - nvqs = netq_num = sc->sc_max_nvq_pairs * 2; - if (sc->sc_has_ctrl) - nvqs++; + snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); + sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, + IPL_NET, WQ_PERCPU | WQ_MPSAFE); + if (sc->sc_txrx_workqueue == NULL) + goto err; - sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); - sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * netq_num, - KM_SLEEP); -} + req_flags = 0; -static void -vioif_free_queues(struct vioif_softc *sc) -{ - size_t nvqs, netq_num; +#ifdef VIOIF_MPSAFE + req_flags |= VIRTIO_F_INTR_MPSAFE; +#endif + req_flags |= VIRTIO_F_INTR_MSIX; - nvqs = netq_num = sc->sc_max_nvq_pairs * 2; - if (sc->sc_ctrlq.ctrlq_vq) - nvqs++; + req_features = + VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | + VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; + req_features |= VIRTIO_F_RING_EVENT_IDX; + req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; +#ifdef VIOIF_MULTIQ + req_features |= VIRTIO_NET_F_MQ; +#endif + virtio_child_attach_start(vsc, self, IPL_NET, NULL, + vioif_config_change, virtio_vq_intrhand, req_flags, + req_features, VIRTIO_NET_FLAG_BITS); - kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num); - kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); - sc->sc_netqs = NULL; - sc->sc_vqs = NULL; -} + features = virtio_features(vsc); + if (features == 0) + goto err; -static int -vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc, - size_t qid, u_int softint_flags) -{ - static const struct { - const char *dirname; - int segsize; - int nsegs; - int (*intrhand)(void *); - void (*sihand)(void *); - } params[VIOIF_NETQ_IDX] = { - [VIOIF_NETQ_RX] = { - .dirname = "rx", - .segsize = MCLBYTES, - .nsegs = 2, - .intrhand = vioif_rx_intr, - .sihand = vioif_rx_handle, - }, - [VIOIF_NETQ_TX] = { - .dirname = "tx", - .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN, - .nsegs = 2, - .intrhand = vioif_tx_intr, - .sihand = vioif_tx_handle, + if (features & VIRTIO_NET_F_MAC) { + for (i = 0; i < __arraycount(sc->sc_mac); i++) { + sc->sc_mac[i] = virtio_read_device_config_1(vsc, + VIRTIO_NET_CONFIG_MAC + i); } - }; + } else { + /* code stolen from sys/net/if_tap.c */ + struct timeval tv; + uint32_t ui; + getmicrouptime(&tv); + ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; + memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); + for (i = 0; i < __arraycount(sc->sc_mac); i++) { + virtio_write_device_config_1(vsc, + VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); + } + } - struct virtqueue *vq; - struct vioif_netqueue *netq; - struct vioif_tx_context *txc; - struct vioif_rx_context *rxc; - char qname[32]; - int r, dir; + /* 'Ethernet' with capital follows other ethernet driver attachment */ + aprint_normal_dev(self, "Ethernet address %s\n", + ether_sprintf(sc->sc_mac)); - txc = NULL; - rxc = NULL; - netq = &sc->sc_netqs[qid]; - vq = &sc->sc_vqs[qid]; - dir = VIOIF_NETQ_DIR(qid); + if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { + sc->sc_hdr_size = sizeof(struct virtio_net_hdr); + } else { + sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); + } - netq->netq_vq = &sc->sc_vqs[qid]; - netq->netq_stopping = false; - netq->netq_running_handle = false; + if ((features & VIRTIO_NET_F_CTRL_VQ) && + (features & VIRTIO_NET_F_CTRL_RX)) { + sc->sc_has_ctrl = true; - snprintf(qname, sizeof(qname), "%s%zu", - params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid)); - snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup), - "%s-%s", device_xname(sc->sc_dev), qname); + cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); + mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); + ctrlq->ctrlq_inuse = FREE; + } else { + sc->sc_has_ctrl = false; + } - mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET); - r = virtio_alloc_vq(vsc, vq, qid, - params[dir].segsize + sc->sc_hdr_size, - params[dir].nsegs, qname); - if (r != 0) - goto err; - netq->netq_vq = vq; + if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { + sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, + VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); - netq->netq_vq->vq_intrhand = params[dir].intrhand; - netq->netq_vq->vq_intrhand_arg = netq; - netq->netq_softint = softint_establish(softint_flags, - params[dir].sihand, netq); - if (netq->netq_softint == NULL) { - aprint_error_dev(sc->sc_dev, - "couldn't establish %s softint\n", - params[dir].dirname); - goto err; + if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) + goto err; + + /* Limit the number of queue pairs to use */ + sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); } - vioif_work_set(&netq->netq_work, params[dir].sihand, netq); - switch (dir) { - case VIOIF_NETQ_RX: - rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP); - netq->netq_ctx = rxc; - /* nothing to do */ - break; - case VIOIF_NETQ_TX: - txc = kmem_zalloc(sizeof(*txc), KM_SLEEP); - netq->netq_ctx = (void *)txc; - txc->txc_deferred_transmit = softint_establish(softint_flags, - vioif_deferred_transmit, netq); - if (txc->txc_deferred_transmit == NULL) { - aprint_error_dev(sc->sc_dev, - "couldn't establish softint for " - "tx deferred transmit\n"); + vioif_alloc_queues(sc); + virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); + +#ifdef VIOIF_MPSAFE + softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; +#else + softint_flags = SOFTINT_NET; +#endif + + /* + * Initialize network queues + */ + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + r = vioif_netqueue_init(sc, vsc, i, softint_flags); + if (r != 0) goto err; + } + + if (sc->sc_has_ctrl) { + int ctrlq_idx = sc->sc_max_nvq_pairs * 2; + /* + * Allocating a virtqueue for control channel + */ + sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx]; + r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx, + NBPG, 1, "control"); + if (r != 0) { + aprint_error_dev(self, "failed to allocate " + "a virtqueue for control channel, error code %d\n", + r); + + sc->sc_has_ctrl = false; + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); + } else { + ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; + ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; } - txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc); - txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP); - break; } - return 0; + sc->sc_cfg_softint = softint_establish(softint_flags, + vioif_cfg_softint, sc); + if (sc->sc_cfg_softint == NULL) { + aprint_error_dev(self, "cannot establish ctl softint\n"); + goto err; + } -err: - netq->netq_ctx = NULL; + if (vioif_alloc_mems(sc) < 0) + goto err; - if (rxc != NULL) { - kmem_free(rxc, sizeof(*rxc)); + if (virtio_child_attach_finish(vsc) != 0) + goto err; + + if (vioif_setup_sysctl(sc) != 0) { + aprint_error_dev(self, "unable to create sysctl node\n"); + /* continue */ } - if (txc != NULL) { - if (txc->txc_deferred_transmit != NULL) - softint_disestablish(txc->txc_deferred_transmit); - if (txc->txc_intrq != NULL) - pcq_destroy(txc->txc_intrq); - kmem_free(txc, sizeof(txc)); + vioif_setup_stats(sc); + + strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); + ifp->if_softc = sc; + ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; +#ifdef VIOIF_MPSAFE + ifp->if_extflags = IFEF_MPSAFE; +#endif + ifp->if_start = vioif_start; + if (sc->sc_req_nvq_pairs > 1) + ifp->if_transmit = vioif_transmit; + ifp->if_ioctl = vioif_ioctl; + ifp->if_init = vioif_init; + ifp->if_stop = vioif_stop; + ifp->if_capabilities = 0; + ifp->if_watchdog = vioif_watchdog; + txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; + IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN)); + IFQ_SET_READY(&ifp->if_snd); + + sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; + + if_attach(ifp); + if_deferred_start_init(ifp, NULL); + ether_ifattach(ifp, sc->sc_mac); + ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); + + return; + +err: + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + vioif_netqueue_teardown(sc, vsc, i); } - vioif_work_set(&netq->netq_work, NULL, NULL); - if (netq->netq_softint != NULL) { - softint_disestablish(netq->netq_softint); - netq->netq_softint = NULL; + if (sc->sc_has_ctrl) { + cv_destroy(&ctrlq->ctrlq_wait); + mutex_destroy(&ctrlq->ctrlq_wait_lock); + virtio_free_vq(vsc, ctrlq->ctrlq_vq); + ctrlq->ctrlq_vq = NULL; } - netq->netq_vq->vq_intrhand = NULL; - netq->netq_vq->vq_intrhand_arg = NULL; - virtio_free_vq(vsc, vq); - mutex_destroy(&netq->netq_lock); - netq->netq_vq = NULL; + vioif_free_queues(sc); + mutex_destroy(&sc->sc_lock); + virtio_child_attach_failed(vsc); + config_finalize_register(self, vioif_finalize_teardown); - return -1; + return; } -static void -vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc, - size_t qid) +static int +vioif_finalize_teardown(device_t self) { - struct vioif_netqueue *netq; - struct vioif_rx_context *rxc; - struct vioif_tx_context *txc; - int dir; - - netq = &sc->sc_netqs[qid]; - - if (netq->netq_vq == NULL) - return; + struct vioif_softc *sc = device_private(self); - netq = &sc->sc_netqs[qid]; - dir = VIOIF_NETQ_DIR(qid); - switch (dir) { - case VIOIF_NETQ_RX: - rxc = netq->netq_ctx; - netq->netq_ctx = NULL; - kmem_free(rxc, sizeof(*rxc)); - break; - case VIOIF_NETQ_TX: - txc = netq->netq_ctx; - netq->netq_ctx = NULL; - softint_disestablish(txc->txc_deferred_transmit); - pcq_destroy(txc->txc_intrq); - kmem_free(txc, sizeof(*txc)); - break; + if (sc->sc_txrx_workqueue != NULL) { + vioif_workq_destroy(sc->sc_txrx_workqueue); + sc->sc_txrx_workqueue = NULL; } - softint_disestablish(netq->netq_softint); - virtio_free_vq(vsc, netq->netq_vq); - mutex_destroy(&netq->netq_lock); - netq->netq_vq = NULL; + return 0; } -/* allocate memory */ /* - * dma memory is used for: - * netq_maps_kva: metadata array for received frames (READ) and - * sent frames (WRITE) - * ctrlq_cmd: command to be sent via ctrl vq (WRITE) - * ctrlq_status: return value for a command via ctrl vq (READ) - * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command - * (WRITE) - * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC - * class command (WRITE) - * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC - * class command (WRITE) - * ctrlq_* structures are allocated only one each; they are protected by - * ctrlq_inuse variable and ctrlq_wait condvar. + * Interface functions for ifnet */ static int -vioif_alloc_mems(struct vioif_softc *sc) +vioif_init(struct ifnet *ifp) { + struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; struct vioif_netqueue *netq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - struct vioif_net_map *maps; - unsigned int vq_num; - int r, rsegs; - bus_size_t dmamemsize; - size_t qid, i, netq_num, kmemsize; - void *vaddr; - intptr_t p; - - netq_num = sc->sc_max_nvq_pairs * 2; + int r, i; - /* allocate DMA memory */ - dmamemsize = 0; + vioif_stop(ifp, 0); - for (qid = 0; qid < netq_num; qid++) { - maps = sc->sc_netqs[qid].netq_maps; - vq_num = sc->sc_netqs[qid].netq_vq->vq_num; - dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num; + r = virtio_reinit_start(vsc); + if (r != 0) { + log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); + return EIO; } - if (sc->sc_has_ctrl) { - dmamemsize += sizeof(struct virtio_net_ctrl_cmd); - dmamemsize += sizeof(struct virtio_net_ctrl_status); - dmamemsize += sizeof(struct virtio_net_ctrl_rx); - dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN; - dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; - dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr); - dmamemsize += sizeof(struct virtio_net_ctrl_mq); - } + virtio_negotiate_features(vsc, virtio_features(vsc)); - r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0, - &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); - if (r != 0) { - aprint_error_dev(sc->sc_dev, - "DMA memory allocation failed, size %zu, " - "error code %d\n", dmamemsize, r); - goto err_none; - } - r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1, - dmamemsize, &vaddr, BUS_DMA_NOWAIT); - if (r != 0) { - aprint_error_dev(sc->sc_dev, - "DMA memory map failed, error code %d\n", r); - goto err_dmamem_alloc; + for (i = 0; i < sc->sc_req_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; + + mutex_enter(&netq->netq_lock); + vioif_populate_rx_mbufs_locked(sc, netq); + mutex_exit(&netq->netq_lock); } - /* assign DMA memory */ - memset(vaddr, 0, dmamemsize); - sc->sc_dmamem = vaddr; - p = (intptr_t) vaddr; + virtio_reinit_end(vsc); - for (qid = 0; qid < netq_num; qid++) { - netq = &sc->sc_netqs[qid]; - maps = netq->netq_maps; - vq_num = netq->netq_vq->vq_num; + if (sc->sc_has_ctrl) + virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); - netq->netq_maps_kva = vioif_assign_mem(&p, - sizeof(*maps[0].vnm_hdr) * vq_num); - } + r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); + if (r == 0) + sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; + else + sc->sc_act_nvq_pairs = 1; - if (sc->sc_has_ctrl) { - ctrlq->ctrlq_cmd = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_cmd)); - ctrlq->ctrlq_status = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_status)); - ctrlq->ctrlq_rx = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_rx)); - ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_tbl_uc) - + ETHER_ADDR_LEN); - ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_tbl_mc) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); - ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, - sizeof(*ctrlq->ctrlq_mac_addr)); - ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); - } + SET(ifp->if_flags, IFF_RUNNING); + CLR(ifp->if_flags, IFF_OACTIVE); - /* allocate kmem */ - kmemsize = 0; + vioif_net_intr_enable(sc, vsc); - for (qid = 0; qid < netq_num; qid++) { - netq = &sc->sc_netqs[qid]; - vq_num = netq->netq_vq->vq_num; + vioif_update_link_status(sc); + r = vioif_rx_filter(sc); - kmemsize += sizeof(netq->netq_maps[0]) * vq_num; - } + return r; +} - vaddr = kmem_zalloc(kmemsize, KM_SLEEP); - sc->sc_kmem = vaddr; +static void +vioif_stop(struct ifnet *ifp, int disable) +{ + struct vioif_softc *sc = ifp->if_softc; + struct virtio_softc *vsc = sc->sc_virtio; + struct vioif_netqueue *netq; + struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; + size_t i, act_qnum; - /* assign allocated kmem */ - p = (intptr_t) vaddr; + act_qnum = sc->sc_act_nvq_pairs * 2; - for (qid = 0; qid < netq_num; qid++) { - netq = &sc->sc_netqs[qid]; - vq_num = netq->netq_vq->vq_num; + CLR(ifp->if_flags, IFF_RUNNING); + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; - netq->netq_maps = vioif_assign_mem(&p, - sizeof(netq->netq_maps[0]) * vq_num); + mutex_enter(&netq->netq_lock); + netq->netq_stopping = true; + mutex_exit(&netq->netq_lock); } - /* prepare dmamaps */ - for (qid = 0; qid < netq_num; qid++) { - static const struct { - const char *msg_hdr; - const char *msg_payload; - int dma_flag; - bus_size_t dma_size; - int dma_nsegs; - } dmaparams[VIOIF_NETQ_IDX] = { - [VIOIF_NETQ_RX] = { - .msg_hdr = "rx header", - .msg_payload = "rx payload", - .dma_flag = BUS_DMA_READ, - .dma_size = MCLBYTES - ETHER_ALIGN, - .dma_nsegs = 1, - }, - [VIOIF_NETQ_TX] = { - .msg_hdr = "tx header", - .msg_payload = "tx payload", - .dma_flag = BUS_DMA_WRITE, - .dma_size = ETHER_MAX_LEN, - .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS, - } - }; - - struct virtio_net_hdr *hdrs; - int dir; + /* disable interrupts */ + vioif_net_intr_disable(sc, vsc); + if (sc->sc_has_ctrl) + virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); - dir = VIOIF_NETQ_DIR(qid); - netq = &sc->sc_netqs[qid]; - vq_num = netq->netq_vq->vq_num; - maps = netq->netq_maps; - hdrs = netq->netq_maps_kva; + /* + * only way to stop interrupt, I/O and DMA is resetting... + * + * NOTE: Devices based on VirtIO draft specification can not + * stop interrupt completely even if virtio_stop_vq_intr() is called. + */ + virtio_reset(vsc); - for (i = 0; i < vq_num; i++) { - maps[i].vnm_hdr = &hdrs[i]; - - r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map, - maps[i].vnm_hdr, sc->sc_hdr_size, 1, - dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr); - if (r != 0) - goto err_reqs; + vioif_intr_barrier(); - r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map, - dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs, - dmaparams[dir].msg_payload); - if (r != 0) - goto err_reqs; - } + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; + vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work); } - if (sc->sc_has_ctrl) { - /* control vq class & command */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, - ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, - BUS_DMA_WRITE, "control command"); - if (r != 0) - goto err_reqs; - - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, - ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, - BUS_DMA_READ, "control status"); - if (r != 0) - goto err_reqs; - - /* control vq rx mode command parameter */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, - ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, - BUS_DMA_WRITE, "rx mode control command"); - if (r != 0) - goto err_reqs; - - /* multiqueue set command */ - r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, - ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, - BUS_DMA_WRITE, "multiqueue set command"); - if (r != 0) - goto err_reqs; - - /* control vq MAC filter table for unicast */ - /* do not load now since its length is variable */ - r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, - sizeof(*ctrlq->ctrlq_mac_tbl_uc) - + ETHER_ADDR_LEN, 1, - "unicast MAC address filter command"); - if (r != 0) - goto err_reqs; - - /* control vq MAC filter table for multicast */ - r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, - sizeof(*ctrlq->ctrlq_mac_tbl_mc) - + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, - "multicast MAC address filter command"); - if (r != 0) - goto err_reqs; + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; + vioif_rx_queue_clear(sc, vsc, netq); - /* control vq MAC address set command */ - r = vioif_dmamap_create_load(sc, - &ctrlq->ctrlq_mac_addr_dmamap, - ctrlq->ctrlq_mac_addr, - sizeof(*ctrlq->ctrlq_mac_addr), 1, - BUS_DMA_WRITE, "mac addr set command"); - if (r != 0) - goto err_reqs; + netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; + vioif_tx_queue_clear(sc, vsc, netq); } - return 0; - -err_reqs: - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); - vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); - for (qid = 0; qid < netq_num; qid++) { - vq_num = sc->sc_netqs[qid].netq_vq->vq_num; - maps = sc->sc_netqs[qid].netq_maps; + /* all packet processing is stopped */ + for (i = 0; i < act_qnum; i++) { + netq = &sc->sc_netqs[i]; - for (i = 0; i < vq_num; i++) { - vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map); - vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map); - } - } - if (sc->sc_kmem) { - kmem_free(sc->sc_kmem, kmemsize); - sc->sc_kmem = NULL; + mutex_enter(&netq->netq_lock); + netq->netq_stopping = false; + mutex_exit(&netq->netq_lock); } - bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize); -err_dmamem_alloc: - bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1); -err_none: - return -1; } static void -vioif_attach(device_t parent, device_t self, void *aux) +vioif_start(struct ifnet *ifp) { - struct vioif_softc *sc = device_private(self); - struct virtio_softc *vsc = device_private(parent); - struct vioif_netqueue *txq0; - struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - uint64_t features, req_features; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - u_int softint_flags; - int r, i, req_flags; - char xnamebuf[MAXCOMLEN]; - size_t netq_num; - - if (virtio_child(vsc) != NULL) { - aprint_normal(": child already attached for %s; " - "something wrong...\n", device_xname(parent)); - return; - } + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; - sc->sc_dev = self; - sc->sc_virtio = vsc; - sc->sc_link_state = LINK_STATE_UNKNOWN; +#ifdef VIOIF_MPSAFE + KASSERT(if_is_mpsafe(ifp)); +#endif - sc->sc_max_nvq_pairs = 1; - sc->sc_req_nvq_pairs = 1; - sc->sc_act_nvq_pairs = 1; - sc->sc_txrx_workqueue_sysctl = true; - sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT; - sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT; - sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT; - sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT; + mutex_enter(&txq0->netq_lock); + vioif_start_locked(ifp, txq0); + mutex_exit(&txq0->netq_lock); +} - mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE); +static inline int +vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + u_int cpuid = cpu_index(curcpu()); - snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self)); - sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI, - IPL_NET, WQ_PERCPU | WQ_MPSAFE); - if (sc->sc_txrx_workqueue == NULL) - goto err; + return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs); +} - req_flags = 0; +static int +vioif_transmit(struct ifnet *ifp, struct mbuf *m) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; + int qid; -#ifdef VIOIF_MPSAFE - req_flags |= VIRTIO_F_INTR_MPSAFE; -#endif - req_flags |= VIRTIO_F_INTR_MSIX; + qid = vioif_select_txqueue(ifp, m); + netq = &sc->sc_netqs[qid]; + txc = netq->netq_ctx; - req_features = - VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ | - VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY; - req_features |= VIRTIO_F_RING_EVENT_IDX; - req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR; -#ifdef VIOIF_MULTIQ - req_features |= VIRTIO_NET_F_MQ; -#endif - virtio_child_attach_start(vsc, self, IPL_NET, NULL, - vioif_config_change, virtio_vq_intrhand, req_flags, - req_features, VIRTIO_NET_FLAG_BITS); + if (__predict_false(!pcq_put(txc->txc_intrq, m))) { + m_freem(m); + return ENOBUFS; + } - features = virtio_features(vsc); - if (features == 0) - goto err; + net_stat_ref_t nsr = IF_STAT_GETREF(ifp); + if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); + if (m->m_flags & M_MCAST) + if_statinc_ref(nsr, if_omcasts); + IF_STAT_PUTREF(ifp); - if (features & VIRTIO_NET_F_MAC) { - for (i = 0; i < __arraycount(sc->sc_mac); i++) { - sc->sc_mac[i] = virtio_read_device_config_1(vsc, - VIRTIO_NET_CONFIG_MAC + i); - } - } else { - /* code stolen from sys/net/if_tap.c */ - struct timeval tv; - uint32_t ui; - getmicrouptime(&tv); - ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff; - memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3); - for (i = 0; i < __arraycount(sc->sc_mac); i++) { - virtio_write_device_config_1(vsc, - VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); - } + if (mutex_tryenter(&netq->netq_lock)) { + vioif_transmit_locked(ifp, netq); + mutex_exit(&netq->netq_lock); } - /* 'Ethernet' with capital follows other ethernet driver attachment */ - aprint_normal_dev(self, "Ethernet address %s\n", - ether_sprintf(sc->sc_mac)); + return 0; +} - if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) { - sc->sc_hdr_size = sizeof(struct virtio_net_hdr); - } else { - sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers); - } +void +vioif_watchdog(struct ifnet *ifp) +{ + struct vioif_softc *sc = ifp->if_softc; + struct vioif_netqueue *netq; + int i; - if ((features & VIRTIO_NET_F_CTRL_VQ) && - (features & VIRTIO_NET_F_CTRL_RX)) { - sc->sc_has_ctrl = true; + if (ifp->if_flags & IFF_RUNNING) { + for (i = 0; i < sc->sc_act_nvq_pairs; i++) { + netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; - cv_init(&ctrlq->ctrlq_wait, "ctrl_vq"); - mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET); - ctrlq->ctrlq_inuse = FREE; - } else { - sc->sc_has_ctrl = false; + mutex_enter(&netq->netq_lock); + if (!netq->netq_running_handle) { + netq->netq_running_handle = true; + vioif_net_sched_handle(sc, netq); + } + mutex_exit(&netq->netq_lock); + } } +} - if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) { - sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc, - VIRTIO_NET_CONFIG_MAX_VQ_PAIRS); +static int +vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) +{ + int s, r; - if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) - goto err; + s = splnet(); - /* Limit the number of queue pairs to use */ - sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu); + r = ether_ioctl(ifp, cmd, data); + if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { + if (ifp->if_flags & IFF_RUNNING) { + r = vioif_rx_filter(ifp->if_softc); + } else { + r = 0; + } } - vioif_alloc_queues(sc); - virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs); + splx(s); -#ifdef VIOIF_MPSAFE - softint_flags = SOFTINT_NET | SOFTINT_MPSAFE; -#else - softint_flags = SOFTINT_NET; -#endif + return r; +} - /* - * Initialize network queues - */ - netq_num = sc->sc_max_nvq_pairs * 2; - for (i = 0; i < netq_num; i++) { - r = vioif_netqueue_init(sc, vsc, i, softint_flags); - if (r != 0) - goto err; - } +static int +vioif_ifflags(struct vioif_softc *sc) +{ + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + bool onoff; + int r; - if (sc->sc_has_ctrl) { - int ctrlq_idx = sc->sc_max_nvq_pairs * 2; - /* - * Allocating a virtqueue for control channel - */ - sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx]; - r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx, - NBPG, 1, "control"); - if (r != 0) { - aprint_error_dev(self, "failed to allocate " - "a virtqueue for control channel, error code %d\n", - r); + if (!sc->sc_has_ctrl) { + /* no ctrl vq; always promisc and allmulti */ + ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); + return 0; + } - sc->sc_has_ctrl = false; - cv_destroy(&ctrlq->ctrlq_wait); - mutex_destroy(&ctrlq->ctrlq_wait_lock); - } else { - ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr; - ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq; + onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; + r = vioif_set_allmulti(sc, onoff); + if (r != 0) { + log(LOG_WARNING, + "%s: couldn't %sable ALLMULTI\n", + ifp->if_xname, onoff ? "en" : "dis"); + if (onoff == false) { + ifp->if_flags |= IFF_ALLMULTI; } } - sc->sc_ctl_softint = softint_establish(softint_flags, - vioif_ctl_softint, sc); - if (sc->sc_ctl_softint == NULL) { - aprint_error_dev(self, "cannot establish ctl softint\n"); - goto err; + onoff = ifp->if_flags & IFF_PROMISC ? true : false; + r = vioif_set_promisc(sc, onoff); + if (r != 0) { + log(LOG_WARNING, + "%s: couldn't %sable PROMISC\n", + ifp->if_xname, onoff ? "en" : "dis"); + if (onoff == false) { + ifp->if_flags |= IFF_PROMISC; + } } - if (vioif_alloc_mems(sc) < 0) - goto err; + return 0; +} - if (virtio_child_attach_finish(vsc) != 0) - goto err; +static int +vioif_ifflags_cb(struct ethercom *ec) +{ + struct ifnet *ifp = &ec->ec_if; + struct vioif_softc *sc = ifp->if_softc; - if (vioif_setup_sysctl(sc) != 0) { - aprint_error_dev(self, "unable to create sysctl node\n"); - /* continue */ - } + return vioif_ifflags(sc); +} - vioif_setup_stats(sc); +static int +vioif_setup_sysctl(struct vioif_softc *sc) +{ + const char *devname; + struct sysctllog **log; + const struct sysctlnode *rnode, *rxnode, *txnode; + int error; + + log = &sc->sc_sysctllog; + devname = device_xname(sc->sc_dev); + + error = sysctl_createv(log, 0, NULL, &rnode, + 0, CTLTYPE_NODE, devname, + SYSCTL_DESCR("virtio-net information and settings"), + NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); + if (error) + goto out; + + error = sysctl_createv(log, 0, &rnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", + SYSCTL_DESCR("Use workqueue for packet processing"), + NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; - strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ); - ifp->if_softc = sc; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; -#ifdef VIOIF_MPSAFE - ifp->if_extflags = IFEF_MPSAFE; -#endif - ifp->if_start = vioif_start; - if (sc->sc_req_nvq_pairs > 1) - ifp->if_transmit = vioif_transmit; - ifp->if_ioctl = vioif_ioctl; - ifp->if_init = vioif_init; - ifp->if_stop = vioif_stop; - ifp->if_capabilities = 0; - ifp->if_watchdog = vioif_watchdog; - txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; - IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN)); - IFQ_SET_READY(&ifp->if_snd); + error = sysctl_createv(log, 0, &rnode, &rxnode, + 0, CTLTYPE_NODE, "rx", + SYSCTL_DESCR("virtio-net information and settings for Rx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; - sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), + NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; - if_attach(ifp); - if_deferred_start_init(ifp, NULL); - ether_ifattach(ifp, sc->sc_mac); - ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb); + error = sysctl_createv(log, 0, &rxnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), + NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; - return; + error = sysctl_createv(log, 0, &rnode, &txnode, + 0, CTLTYPE_NODE, "tx", + SYSCTL_DESCR("virtio-net information and settings for Tx"), + NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; -err: - netq_num = sc->sc_max_nvq_pairs * 2; - for (i = 0; i < netq_num; i++) { - vioif_netqueue_teardown(sc, vsc, i); - } + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", + SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), + NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); + if (error) + goto out; - if (sc->sc_has_ctrl) { - cv_destroy(&ctrlq->ctrlq_wait); - mutex_destroy(&ctrlq->ctrlq_wait_lock); - virtio_free_vq(vsc, ctrlq->ctrlq_vq); - ctrlq->ctrlq_vq = NULL; - } + error = sysctl_createv(log, 0, &txnode, NULL, + CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", + SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), + NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); - vioif_free_queues(sc); - mutex_destroy(&sc->sc_lock); - virtio_child_attach_failed(vsc); - config_finalize_register(self, vioif_finalize_teardown); +out: + if (error) + sysctl_teardown(log); - return; + return error; } -static int -vioif_finalize_teardown(device_t self) +static void +vioif_setup_stats(struct vioif_softc *sc) { - struct vioif_softc *sc = device_private(self); + struct vioif_netqueue *netq; + struct vioif_tx_context *txc; + struct vioif_rx_context *rxc; + size_t i, netq_num; - if (sc->sc_txrx_workqueue != NULL) { - vioif_workq_destroy(sc->sc_txrx_workqueue); - sc->sc_txrx_workqueue = NULL; + netq_num = sc->sc_max_nvq_pairs * 2; + for (i = 0; i < netq_num; i++) { + netq = &sc->sc_netqs[i]; + evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC, + NULL, netq->netq_evgroup, "failed to load mbuf to DMA"); + evcnt_attach_dynamic(&netq->netq_enqueue_failed, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "virtqueue enqueue failed failed"); + + switch (VIOIF_NETQ_DIR(i)) { + case VIOIF_NETQ_RX: + rxc = netq->netq_ctx; + evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "no receive buffer"); + break; + case VIOIF_NETQ_TX: + txc = netq->netq_ctx; + evcnt_attach_dynamic(&txc->txc_defrag_failed, + EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, + "m_defrag() failed"); + break; + } } - return 0; + evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, + NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); + evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, + NULL, device_xname(sc->sc_dev), "control command failed"); } /* - * Interface functions for ifnet + * allocate memory */ static int -vioif_init(struct ifnet *ifp) +vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map, + bus_size_t size, int nsegs, const char *usage) { - struct vioif_softc *sc = ifp->if_softc; - struct virtio_softc *vsc = sc->sc_virtio; - struct vioif_netqueue *netq; - struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - int r, i; + int r; - vioif_stop(ifp, 0); + r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size, + nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map); - r = virtio_reinit_start(vsc); if (r != 0) { - log(LOG_ERR, "%s: reset failed\n", ifp->if_xname); - return EIO; + aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, " + "error code %d\n", usage, r); } - virtio_negotiate_features(vsc, virtio_features(vsc)); + return r; +} - for (i = 0; i < sc->sc_req_nvq_pairs; i++) { - netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; +static void +vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map) +{ - mutex_enter(&netq->netq_lock); - vioif_populate_rx_mbufs_locked(sc, netq); - mutex_exit(&netq->netq_lock); + if (*map) { + bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map); + *map = NULL; } +} - virtio_reinit_end(vsc); +static int +vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map, + void *buf, bus_size_t size, int nsegs, int rw, const char *usage) +{ + int r; - if (sc->sc_has_ctrl) - virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq); + r = vioif_dmamap_create(sc, map, size, nsegs, usage); + if (r != 0) + return 1; - r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs); - if (r == 0) - sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs; - else - sc->sc_act_nvq_pairs = 1; + r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf, + size, NULL, rw | BUS_DMA_NOWAIT); + if (r != 0) { + vioif_dmamap_destroy(sc, map); + aprint_error_dev(sc->sc_dev, "%s dmamap load failed. " + "error code %d\n", usage, r); + } - SET(ifp->if_flags, IFF_RUNNING); - CLR(ifp->if_flags, IFF_OACTIVE); + return r; +} - vioif_net_intr_enable(sc, vsc); +static void * +vioif_assign_mem(intptr_t *p, size_t size) +{ + intptr_t rv; - vioif_update_link_status(sc); - r = vioif_rx_filter(sc); + rv = *p; + *p += size; - return r; + return (void *)rv; } -static void -vioif_stop(struct ifnet *ifp, int disable) +/* + * dma memory is used for: + * netq_maps_kva: metadata array for received frames (READ) and + * sent frames (WRITE) + * ctrlq_cmd: command to be sent via ctrl vq (WRITE) + * ctrlq_status: return value for a command via ctrl vq (READ) + * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command + * (WRITE) + * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * class command (WRITE) + * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC + * class command (WRITE) + * ctrlq_* structures are allocated only one each; they are protected by + * ctrlq_inuse variable and ctrlq_wait condvar. + */ +static int +vioif_alloc_mems(struct vioif_softc *sc) { - struct vioif_softc *sc = ifp->if_softc; struct virtio_softc *vsc = sc->sc_virtio; struct vioif_netqueue *netq; struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq; - size_t i, act_qnum; + struct vioif_net_map *maps; + unsigned int vq_num; + int r, rsegs; + bus_size_t dmamemsize; + size_t qid, i, netq_num, kmemsize; + void *vaddr; + intptr_t p; + + netq_num = sc->sc_max_nvq_pairs * 2; + + /* allocate DMA memory */ + dmamemsize = 0; + + for (qid = 0; qid < netq_num; qid++) { + maps = sc->sc_netqs[qid].netq_maps; + vq_num = sc->sc_netqs[qid].netq_vq->vq_num; + dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num; + } + + if (sc->sc_has_ctrl) { + dmamemsize += sizeof(struct virtio_net_ctrl_cmd); + dmamemsize += sizeof(struct virtio_net_ctrl_status); + dmamemsize += sizeof(struct virtio_net_ctrl_rx); + dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) + + ETHER_ADDR_LEN; + dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES; + dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr); + dmamemsize += sizeof(struct virtio_net_ctrl_mq); + } + + r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0, + &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); + if (r != 0) { + aprint_error_dev(sc->sc_dev, + "DMA memory allocation failed, size %zu, " + "error code %d\n", dmamemsize, r); + goto err_none; + } + r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1, + dmamemsize, &vaddr, BUS_DMA_NOWAIT); + if (r != 0) { + aprint_error_dev(sc->sc_dev, + "DMA memory map failed, error code %d\n", r); + goto err_dmamem_alloc; + } - act_qnum = sc->sc_act_nvq_pairs * 2; + /* assign DMA memory */ + memset(vaddr, 0, dmamemsize); + sc->sc_dmamem = vaddr; + p = (intptr_t) vaddr; - CLR(ifp->if_flags, IFF_RUNNING); - for (i = 0; i < act_qnum; i++) { - netq = &sc->sc_netqs[i]; + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + maps = netq->netq_maps; + vq_num = netq->netq_vq->vq_num; - mutex_enter(&netq->netq_lock); - netq->netq_stopping = true; - mutex_exit(&netq->netq_lock); + netq->netq_maps_kva = vioif_assign_mem(&p, + sizeof(*maps[0].vnm_hdr) * vq_num); } - /* disable interrupts */ - vioif_net_intr_disable(sc, vsc); - if (sc->sc_has_ctrl) - virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq); + if (sc->sc_has_ctrl) { + ctrlq->ctrlq_cmd = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_cmd)); + ctrlq->ctrlq_status = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_status)); + ctrlq->ctrlq_rx = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_rx)); + ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_tbl_uc) + + ETHER_ADDR_LEN); + ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES); + ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p, + sizeof(*ctrlq->ctrlq_mac_addr)); + ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq)); + } - /* - * only way to stop interrupt, I/O and DMA is resetting... - * - * NOTE: Devices based on VirtIO draft specification can not - * stop interrupt completely even if virtio_stop_vq_intr() is called. - */ - virtio_reset(vsc); + /* allocate kmem */ + kmemsize = 0; - vioif_intr_barrier(); + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; - for (i = 0; i < act_qnum; i++) { - netq = &sc->sc_netqs[i]; - vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work); + kmemsize += sizeof(netq->netq_maps[0]) * vq_num; } - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)]; - vioif_rx_queue_clear(sc, vsc, netq); + vaddr = kmem_zalloc(kmemsize, KM_SLEEP); + sc->sc_kmem = vaddr; - netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; - vioif_tx_queue_clear(sc, vsc, netq); - } + /* assign allocated kmem */ + p = (intptr_t) vaddr; - /* all packet processing is stopped */ - for (i = 0; i < act_qnum; i++) { - netq = &sc->sc_netqs[i]; + for (qid = 0; qid < netq_num; qid++) { + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; - mutex_enter(&netq->netq_lock); - netq->netq_stopping = false; - mutex_exit(&netq->netq_lock); + netq->netq_maps = vioif_assign_mem(&p, + sizeof(netq->netq_maps[0]) * vq_num); } -} -static void -vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq, - bool is_transmit) -{ - struct vioif_softc *sc = ifp->if_softc; - struct virtio_softc *vsc = sc->sc_virtio; - struct virtqueue *vq = netq->netq_vq; - struct vioif_tx_context *txc; - struct vioif_net_map *map; - struct mbuf *m; - int queued = 0; - - KASSERT(mutex_owned(&netq->netq_lock)); + /* prepare dmamaps */ + for (qid = 0; qid < netq_num; qid++) { + static const struct { + const char *msg_hdr; + const char *msg_payload; + int dma_flag; + bus_size_t dma_size; + int dma_nsegs; + } dmaparams[VIOIF_NETQ_IDX] = { + [VIOIF_NETQ_RX] = { + .msg_hdr = "rx header", + .msg_payload = "rx payload", + .dma_flag = BUS_DMA_READ, + .dma_size = MCLBYTES - ETHER_ALIGN, + .dma_nsegs = 1, + }, + [VIOIF_NETQ_TX] = { + .msg_hdr = "tx header", + .msg_payload = "tx payload", + .dma_flag = BUS_DMA_WRITE, + .dma_size = ETHER_MAX_LEN, + .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS, + } + }; - if (netq->netq_stopping || - !ISSET(ifp->if_flags, IFF_RUNNING)) - return; + struct virtio_net_hdr *hdrs; + int dir; - txc = netq->netq_ctx; + dir = VIOIF_NETQ_DIR(qid); + netq = &sc->sc_netqs[qid]; + vq_num = netq->netq_vq->vq_num; + maps = netq->netq_maps; + hdrs = netq->netq_maps_kva; - if (!txc->txc_link_active) - return; + for (i = 0; i < vq_num; i++) { + maps[i].vnm_hdr = &hdrs[i]; + + r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map, + maps[i].vnm_hdr, sc->sc_hdr_size, 1, + dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr); + if (r != 0) + goto err_reqs; - if (!is_transmit && - ISSET(ifp->if_flags, IFF_OACTIVE)) - return; + r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map, + dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs, + dmaparams[dir].msg_payload); + if (r != 0) + goto err_reqs; + } + } - for (;;) { - int slot, r; - r = virtio_enqueue_prep(vsc, vq, &slot); - if (r == EAGAIN) - break; - if (__predict_false(r != 0)) - panic("enqueue_prep for tx buffers"); + if (sc->sc_has_ctrl) { + /* control vq class & command */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap, + ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1, + BUS_DMA_WRITE, "control command"); + if (r != 0) + goto err_reqs; - if (is_transmit) - m = pcq_get(txc->txc_intrq); - else - IFQ_DEQUEUE(&ifp->if_snd, m); + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap, + ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1, + BUS_DMA_READ, "control status"); + if (r != 0) + goto err_reqs; - if (m == NULL) { - virtio_enqueue_abort(vsc, vq, slot); - break; - } + /* control vq rx mode command parameter */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap, + ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1, + BUS_DMA_WRITE, "rx mode control command"); + if (r != 0) + goto err_reqs; - map = &netq->netq_maps[slot]; - KASSERT(map->vnm_mbuf == NULL); + /* multiqueue set command */ + r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap, + ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1, + BUS_DMA_WRITE, "multiqueue set command"); + if (r != 0) + goto err_reqs; - r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE); - if (r != 0) { - /* maybe just too fragmented */ - struct mbuf *newm; + /* control vq MAC filter table for unicast */ + /* do not load now since its length is variable */ + r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_uc) + + ETHER_ADDR_LEN, 1, + "unicast MAC address filter command"); + if (r != 0) + goto err_reqs; - newm = m_defrag(m, M_NOWAIT); - if (newm != NULL) { - m = newm; - r = vioif_net_load_mbuf(vsc, map, m, - BUS_DMA_WRITE); - } else { - txc->txc_defrag_failed.ev_count++; - r = -1; - } + /* control vq MAC filter table for multicast */ + r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap, + sizeof(*ctrlq->ctrlq_mac_tbl_mc) + + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1, + "multicast MAC address filter command"); + if (r != 0) + goto err_reqs; - if (r != 0) { - netq->netq_mbuf_load_failed.ev_count++; - m_freem(m); - if_statinc(ifp, if_oerrors); - virtio_enqueue_abort(vsc, vq, slot); - continue; - } - } + /* control vq MAC address set command */ + r = vioif_dmamap_create_load(sc, + &ctrlq->ctrlq_mac_addr_dmamap, + ctrlq->ctrlq_mac_addr, + sizeof(*ctrlq->ctrlq_mac_addr), 1, + BUS_DMA_WRITE, "mac addr set command"); + if (r != 0) + goto err_reqs; + } - memset(map->vnm_hdr, 0, sc->sc_hdr_size); + return 0; - r = vioif_net_enqueue_tx(vsc, vq, slot, map); - if (r != 0) { - netq->netq_enqueue_failed.ev_count++; - vioif_net_unload_mbuf(vsc, map); - m_freem(m); - /* slot already freed by vioif_net_enqueue_tx */ +err_reqs: + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap); + vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap); + for (qid = 0; qid < netq_num; qid++) { + vq_num = sc->sc_netqs[qid].netq_vq->vq_num; + maps = sc->sc_netqs[qid].netq_maps; - if_statinc(ifp, if_oerrors); - continue; + for (i = 0; i < vq_num; i++) { + vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map); + vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map); } - - queued++; - bpf_mtap(ifp, m, BPF_D_OUT); } - - if (queued > 0) { - vioif_notify(vsc, vq); - ifp->if_timer = 5; + if (sc->sc_kmem) { + kmem_free(sc->sc_kmem, kmemsize); + sc->sc_kmem = NULL; } + bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize); +err_dmamem_alloc: + bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1); +err_none: + return -1; } static void -vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq) -{ - - /* - * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. - */ - vioif_send_common_locked(ifp, netq, false); - -} - -static void -vioif_start(struct ifnet *ifp) +vioif_alloc_queues(struct vioif_softc *sc) { - struct vioif_softc *sc = ifp->if_softc; - struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]; - -#ifdef VIOIF_MPSAFE - KASSERT(if_is_mpsafe(ifp)); -#endif + int nvq_pairs = sc->sc_max_nvq_pairs; + size_t nvqs, netq_num; - mutex_enter(&txq0->netq_lock); - vioif_start_locked(ifp, txq0); - mutex_exit(&txq0->netq_lock); -} + KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX); -static inline int -vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m) -{ - struct vioif_softc *sc = ifp->if_softc; - u_int cpuid = cpu_index(curcpu()); + nvqs = netq_num = sc->sc_max_nvq_pairs * 2; + if (sc->sc_has_ctrl) + nvqs++; - return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs); + sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP); + sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * netq_num, + KM_SLEEP); } static void -vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq) +vioif_free_queues(struct vioif_softc *sc) { + size_t nvqs, netq_num; - vioif_send_common_locked(ifp, netq, true); + nvqs = netq_num = sc->sc_max_nvq_pairs * 2; + if (sc->sc_ctrlq.ctrlq_vq) + nvqs++; + + kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num); + kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs); + sc->sc_netqs = NULL; + sc->sc_vqs = NULL; } +/* + * Network queues + */ static int -vioif_transmit(struct ifnet *ifp, struct mbuf *m) +vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc, + size_t qid, u_int softint_flags) { - struct vioif_softc *sc = ifp->if_softc; + static const struct { + const char *dirname; + int segsize; + int nsegs; + int (*intrhand)(void *); + void (*sihand)(void *); + } params[VIOIF_NETQ_IDX] = { + [VIOIF_NETQ_RX] = { + .dirname = "rx", + .segsize = MCLBYTES, + .nsegs = 2, + .intrhand = vioif_rx_intr, + .sihand = vioif_rx_handle, + }, + [VIOIF_NETQ_TX] = { + .dirname = "tx", + .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN, + .nsegs = 2, + .intrhand = vioif_tx_intr, + .sihand = vioif_tx_handle, + } + }; + + struct virtqueue *vq; struct vioif_netqueue *netq; struct vioif_tx_context *txc; - int qid; + struct vioif_rx_context *rxc; + char qname[32]; + int r, dir; - qid = vioif_select_txqueue(ifp, m); + txc = NULL; + rxc = NULL; netq = &sc->sc_netqs[qid]; - txc = netq->netq_ctx; + vq = &sc->sc_vqs[qid]; + dir = VIOIF_NETQ_DIR(qid); - if (__predict_false(!pcq_put(txc->txc_intrq, m))) { - m_freem(m); - return ENOBUFS; - } + netq->netq_vq = &sc->sc_vqs[qid]; + netq->netq_stopping = false; + netq->netq_running_handle = false; - net_stat_ref_t nsr = IF_STAT_GETREF(ifp); - if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len); - if (m->m_flags & M_MCAST) - if_statinc_ref(nsr, if_omcasts); - IF_STAT_PUTREF(ifp); + snprintf(qname, sizeof(qname), "%s%zu", + params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid)); + snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup), + "%s-%s", device_xname(sc->sc_dev), qname); - if (mutex_tryenter(&netq->netq_lock)) { - vioif_transmit_locked(ifp, netq); - mutex_exit(&netq->netq_lock); + mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET); + r = virtio_alloc_vq(vsc, vq, qid, + params[dir].segsize + sc->sc_hdr_size, + params[dir].nsegs, qname); + if (r != 0) + goto err; + netq->netq_vq = vq; + + netq->netq_vq->vq_intrhand = params[dir].intrhand; + netq->netq_vq->vq_intrhand_arg = netq; + netq->netq_softint = softint_establish(softint_flags, + params[dir].sihand, netq); + if (netq->netq_softint == NULL) { + aprint_error_dev(sc->sc_dev, + "couldn't establish %s softint\n", + params[dir].dirname); + goto err; } + vioif_work_set(&netq->netq_work, params[dir].sihand, netq); - return 0; -} + switch (dir) { + case VIOIF_NETQ_RX: + rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP); + netq->netq_ctx = rxc; + /* nothing to do */ + break; + case VIOIF_NETQ_TX: + txc = kmem_zalloc(sizeof(*txc), KM_SLEEP); + netq->netq_ctx = (void *)txc; + txc->txc_deferred_transmit = softint_establish(softint_flags, + vioif_deferred_transmit, netq); + if (txc->txc_deferred_transmit == NULL) { + aprint_error_dev(sc->sc_dev, + "couldn't establish softint for " + "tx deferred transmit\n"); + goto err; + } + txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc); + txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP); + break; + } -static void -vioif_deferred_transmit(void *arg) -{ - struct vioif_netqueue *netq = arg; - struct virtio_softc *vsc = netq->netq_vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - struct ifnet *ifp = &sc->sc_ethercom.ec_if; + return 0; - mutex_enter(&netq->netq_lock); - vioif_send_common_locked(ifp, netq, true); - mutex_exit(&netq->netq_lock); -} +err: + netq->netq_ctx = NULL; -static int -vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data) -{ - int s, r; + if (rxc != NULL) { + kmem_free(rxc, sizeof(*rxc)); + } - s = splnet(); + if (txc != NULL) { + if (txc->txc_deferred_transmit != NULL) + softint_disestablish(txc->txc_deferred_transmit); + if (txc->txc_intrq != NULL) + pcq_destroy(txc->txc_intrq); + kmem_free(txc, sizeof(txc)); + } - r = ether_ioctl(ifp, cmd, data); - if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) { - if (ifp->if_flags & IFF_RUNNING) { - r = vioif_rx_filter(ifp->if_softc); - } else { - r = 0; - } + vioif_work_set(&netq->netq_work, NULL, NULL); + if (netq->netq_softint != NULL) { + softint_disestablish(netq->netq_softint); + netq->netq_softint = NULL; } + netq->netq_vq->vq_intrhand = NULL; + netq->netq_vq->vq_intrhand_arg = NULL; - splx(s); + virtio_free_vq(vsc, vq); + mutex_destroy(&netq->netq_lock); + netq->netq_vq = NULL; - return r; + return -1; } -void -vioif_watchdog(struct ifnet *ifp) +static void +vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc, + size_t qid) { - struct vioif_softc *sc = ifp->if_softc; struct vioif_netqueue *netq; - int i; + struct vioif_rx_context *rxc; + struct vioif_tx_context *txc; + int dir; - if (ifp->if_flags & IFF_RUNNING) { - for (i = 0; i < sc->sc_act_nvq_pairs; i++) { - netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)]; + netq = &sc->sc_netqs[qid]; - mutex_enter(&netq->netq_lock); - if (!netq->netq_running_handle) { - netq->netq_running_handle = true; - vioif_net_sched_handle(sc, netq); - } - mutex_exit(&netq->netq_lock); - } + if (netq->netq_vq == NULL) + return; + + netq = &sc->sc_netqs[qid]; + dir = VIOIF_NETQ_DIR(qid); + switch (dir) { + case VIOIF_NETQ_RX: + rxc = netq->netq_ctx; + netq->netq_ctx = NULL; + kmem_free(rxc, sizeof(*rxc)); + break; + case VIOIF_NETQ_TX: + txc = netq->netq_ctx; + netq->netq_ctx = NULL; + softint_disestablish(txc->txc_deferred_transmit); + pcq_destroy(txc->txc_intrq); + kmem_free(txc, sizeof(*txc)); + break; } + + softint_disestablish(netq->netq_softint); + virtio_free_vq(vsc, netq->netq_vq); + mutex_destroy(&netq->netq_lock); + netq->netq_vq = NULL; } static void @@ -1613,13 +1655,6 @@ vioif_net_enqueue_rx(struct virtio_softc BUS_DMASYNC_PREREAD, false); } -static void -vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq) -{ - - virtio_enqueue_commit(vsc, vq, -1, true); -} - static struct mbuf * vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq, int slot, struct vioif_net_map *map, int dma_flags) @@ -1679,13 +1714,12 @@ vioif_net_intr_disable(struct vioif_soft virtio_stop_vq_intr(vsc, netq->netq_vq); } - } /* * Receive implementation */ -/* add mbufs for all the empty receive slots */ +/* enqueue mbufs to receive slots */ static void vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq) { @@ -1750,37 +1784,6 @@ vioif_populate_rx_mbufs_locked(struct vi vioif_notify(vsc, vq); } -static void -vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, - struct vioif_netqueue *netq) -{ - struct vioif_net_map *map; - struct mbuf *m; - unsigned int i, vq_num; - bool more; - - mutex_enter(&netq->netq_lock); - vq_num = netq->netq_vq->vq_num; - - for (;;) { - more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL); - if (more == false) - break; - } - - for (i = 0; i < vq_num; i++) { - map = &netq->netq_maps[i]; - - m = map->vnm_mbuf; - if (m == NULL) - continue; - - vioif_net_unload_mbuf(vsc, map); - m_freem(m); - } - mutex_exit(&netq->netq_lock); -} - /* dequeue received packets */ static bool vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, @@ -1829,7 +1832,36 @@ done: return more; } -/* rx interrupt; call _dequeue above and schedule a softint */ +static void +vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq) +{ + struct vioif_net_map *map; + struct mbuf *m; + unsigned int i, vq_num; + bool more; + + mutex_enter(&netq->netq_lock); + + vq_num = netq->netq_vq->vq_num; + for (;;) { + more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL); + if (more == false) + break; + } + + for (i = 0; i < vq_num; i++) { + map = &netq->netq_maps[i]; + + m = map->vnm_mbuf; + if (m == NULL) + continue; + + vioif_net_unload_mbuf(vsc, map); + m_freem(m); + } + mutex_exit(&netq->netq_lock); +} static void vioif_rx_handle_locked(void *xnetq, u_int limit) @@ -1854,77 +1886,267 @@ vioif_rx_handle_locked(void *xnetq, u_in return; } - enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); - if (enqueued != 0) { - virtio_stop_vq_intr(vsc, netq->netq_vq); - vioif_net_sched_handle(sc, netq); - return; + enqueued = virtio_start_vq_intr(vsc, netq->netq_vq); + if (enqueued != 0) { + virtio_stop_vq_intr(vsc, netq->netq_vq); + vioif_net_sched_handle(sc, netq); + return; + } + + netq->netq_running_handle = false; +} + +static int +vioif_rx_intr(void *arg) +{ + struct vioif_netqueue *netq = arg; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + u_int limit; + + mutex_enter(&netq->netq_lock); + + /* handler is already running in softint/workqueue */ + if (netq->netq_running_handle) + goto done; + + netq->netq_running_handle = true; + + limit = sc->sc_rx_intr_process_limit; + virtio_stop_vq_intr(vsc, vq); + vioif_rx_handle_locked(netq, limit); + +done: + mutex_exit(&netq->netq_lock); + return 1; +} + +static void +vioif_rx_handle(void *xnetq) +{ + struct vioif_netqueue *netq = xnetq; + struct virtqueue *vq = netq->netq_vq; + struct virtio_softc *vsc = vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + u_int limit; + + mutex_enter(&netq->netq_lock); + + KASSERT(netq->netq_running_handle); + + if (netq->netq_stopping) { + netq->netq_running_handle = false; + goto done; + } + + limit = sc->sc_rx_process_limit; + vioif_rx_handle_locked(netq, limit); + +done: + mutex_exit(&netq->netq_lock); +} + +/* + * Transmition implementation + */ +/* enqueue mbufs to send */ +static void +vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq, + bool is_transmit) +{ + struct vioif_softc *sc = ifp->if_softc; + struct virtio_softc *vsc = sc->sc_virtio; + struct virtqueue *vq = netq->netq_vq; + struct vioif_tx_context *txc; + struct vioif_net_map *map; + struct mbuf *m; + int queued = 0; + + KASSERT(mutex_owned(&netq->netq_lock)); + + if (netq->netq_stopping || + !ISSET(ifp->if_flags, IFF_RUNNING)) + return; + + txc = netq->netq_ctx; + + if (!txc->txc_link_active) + return; + + if (!is_transmit && + ISSET(ifp->if_flags, IFF_OACTIVE)) + return; + + for (;;) { + int slot, r; + r = virtio_enqueue_prep(vsc, vq, &slot); + if (r == EAGAIN) + break; + if (__predict_false(r != 0)) + panic("enqueue_prep for tx buffers"); + + if (is_transmit) + m = pcq_get(txc->txc_intrq); + else + IFQ_DEQUEUE(&ifp->if_snd, m); + + if (m == NULL) { + virtio_enqueue_abort(vsc, vq, slot); + break; + } + + map = &netq->netq_maps[slot]; + KASSERT(map->vnm_mbuf == NULL); + + r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE); + if (r != 0) { + /* maybe just too fragmented */ + struct mbuf *newm; + + newm = m_defrag(m, M_NOWAIT); + if (newm != NULL) { + m = newm; + r = vioif_net_load_mbuf(vsc, map, m, + BUS_DMA_WRITE); + } else { + txc->txc_defrag_failed.ev_count++; + r = -1; + } + + if (r != 0) { + netq->netq_mbuf_load_failed.ev_count++; + m_freem(m); + if_statinc(ifp, if_oerrors); + virtio_enqueue_abort(vsc, vq, slot); + continue; + } + } + + memset(map->vnm_hdr, 0, sc->sc_hdr_size); + + r = vioif_net_enqueue_tx(vsc, vq, slot, map); + if (r != 0) { + netq->netq_enqueue_failed.ev_count++; + vioif_net_unload_mbuf(vsc, map); + m_freem(m); + /* slot already freed by vioif_net_enqueue_tx */ + + if_statinc(ifp, if_oerrors); + continue; + } + + queued++; + bpf_mtap(ifp, m, BPF_D_OUT); + } + + if (queued > 0) { + vioif_notify(vsc, vq); + ifp->if_timer = 5; + } +} + +/* dequeue sent mbufs */ +static bool +vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq, u_int limit) +{ + struct virtqueue *vq = netq->netq_vq; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_net_map *map; + struct mbuf *m; + int slot, len; + bool more = false; + + KASSERT(mutex_owned(&netq->netq_lock)); + + if (virtio_vq_is_enqueued(vsc, vq) == false) + return false; + + for (;;) { + if (limit-- == 0) { + more = true; + break; + } + + if (virtio_dequeue(vsc, vq, &slot, &len) != 0) + break; + + map = &netq->netq_maps[slot]; + KASSERT(map->vnm_mbuf != NULL); + m = vioif_net_dequeue_commit(vsc, vq, slot, + map, BUS_DMASYNC_POSTWRITE); + KASSERT(m != NULL); + + if_statinc(ifp, if_opackets); + m_freem(m); } - netq->netq_running_handle = false; + return more; } -static int -vioif_rx_intr(void *arg) +static void +vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, + struct vioif_netqueue *netq) { - struct vioif_netqueue *netq = arg; - struct virtqueue *vq = netq->netq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit; + struct vioif_net_map *map; + struct mbuf *m; + unsigned int i, vq_num; + bool more; mutex_enter(&netq->netq_lock); - /* handler is already running in softint/workqueue */ - if (netq->netq_running_handle) - goto done; + vq_num = netq->netq_vq->vq_num; + for (;;) { + more = vioif_tx_deq_locked(sc, vsc, netq, vq_num); + if (more == false) + break; + } - netq->netq_running_handle = true; + for (i = 0; i < vq_num; i++) { + map = &netq->netq_maps[i]; - limit = sc->sc_rx_intr_process_limit; - virtio_stop_vq_intr(vsc, vq); - vioif_rx_handle_locked(netq, limit); + m = map->vnm_mbuf; + if (m == NULL) + continue; -done: + vioif_net_unload_mbuf(vsc, map); + m_freem(m); + } mutex_exit(&netq->netq_lock); - return 1; } static void -vioif_rx_handle(void *xnetq) +vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq) { - struct vioif_netqueue *netq = xnetq; - struct virtqueue *vq = netq->netq_vq; - struct virtio_softc *vsc = vq->vq_owner; - struct vioif_softc *sc = device_private(virtio_child(vsc)); - u_int limit; - mutex_enter(&netq->netq_lock); + /* + * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c. + */ + vioif_send_common_locked(ifp, netq, false); - KASSERT(netq->netq_running_handle); +} - if (netq->netq_stopping) { - netq->netq_running_handle = false; - goto done; - } +static void +vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq) +{ - limit = sc->sc_rx_process_limit; - vioif_rx_handle_locked(netq, limit); + vioif_send_common_locked(ifp, netq, true); +} -done: +static void +vioif_deferred_transmit(void *arg) +{ + struct vioif_netqueue *netq = arg; + struct virtio_softc *vsc = netq->netq_vq->vq_owner; + struct vioif_softc *sc = device_private(virtio_child(vsc)); + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + + mutex_enter(&netq->netq_lock); + vioif_send_common_locked(ifp, netq, true); mutex_exit(&netq->netq_lock); } -/* - * Transmition implementation - */ -/* actual transmission is done in if_start */ -/* tx interrupt; dequeue and free mbufs */ -/* - * tx interrupt is actually disabled; this should be called upon - * tx vq full and watchdog - */ - static void vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit) { @@ -2019,75 +2241,6 @@ done: mutex_exit(&netq->netq_lock); } -static void -vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc, - struct vioif_netqueue *netq) -{ - struct vioif_net_map *map; - struct mbuf *m; - unsigned int i, vq_num; - bool more; - - mutex_enter(&netq->netq_lock); - - vq_num = netq->netq_vq->vq_num; - for (;;) { - more = vioif_tx_deq_locked(sc, vsc, netq, vq_num); - if (more == false) - break; - } - - for (i = 0; i < vq_num; i++) { - map = &netq->netq_maps[i]; - - m = map->vnm_mbuf; - if (m == NULL) - continue; - - vioif_net_unload_mbuf(vsc, map); - m_freem(m); - } - mutex_exit(&netq->netq_lock); -} - -static bool -vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc, - struct vioif_netqueue *netq, u_int limit) -{ - struct virtqueue *vq = netq->netq_vq; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - struct vioif_net_map *map; - struct mbuf *m; - int slot, len; - bool more = false; - - KASSERT(mutex_owned(&netq->netq_lock)); - - if (virtio_vq_is_enqueued(vsc, vq) == false) - return false; - - for (;;) { - if (limit-- == 0) { - more = true; - break; - } - - if (virtio_dequeue(vsc, vq, &slot, &len) != 0) - break; - - map = &netq->netq_maps[slot]; - KASSERT(map->vnm_mbuf != NULL); - m = vioif_net_dequeue_commit(vsc, vq, slot, - map, BUS_DMASYNC_POSTWRITE); - KASSERT(m != NULL); - - if_statinc(ifp, if_opackets); - m_freem(m); - } - - return more; -} - /* * Control vq */ @@ -2227,6 +2380,31 @@ vioif_ctrl_send_command(struct vioif_sof return r; } +/* ctrl vq interrupt; wake up the command issuer */ +static int +vioif_ctrl_intr(void *arg) +{ + struct vioif_ctrlqueue *ctrlq = arg; + struct virtqueue *vq = ctrlq->ctrlq_vq; + struct virtio_softc *vsc = vq->vq_owner; + int r, slot; + + if (virtio_vq_is_enqueued(vsc, vq) == false) + return 0; + + r = virtio_dequeue(vsc, vq, &slot, NULL); + if (r == ENOENT) + return 0; + virtio_dequeue_commit(vsc, vq, slot); + + mutex_enter(&ctrlq->ctrlq_wait_lock); + ctrlq->ctrlq_inuse = DONE; + cv_signal(&ctrlq->ctrlq_wait); + mutex_exit(&ctrlq->ctrlq_wait_lock); + + return 1; +} + static int vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff) { @@ -2263,46 +2441,30 @@ vioif_set_allmulti(struct vioif_softc *s return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff); } -/* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */ static int -vioif_set_rx_filter(struct vioif_softc *sc) +vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) { - /* filter already set in ctrlq->ctrlq_mac_tbl */ - struct virtio_softc *vsc = sc->sc_virtio; - struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; - struct vioif_ctrl_cmdspec specs[2]; - int nspecs = __arraycount(specs); + struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; + struct vioif_ctrl_cmdspec specs[1]; int r; - mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; - mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; - if (!sc->sc_has_ctrl) return ENOTSUP; - vioif_ctrl_acquire(sc); - - specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; - specs[0].buf = mac_tbl_uc; - specs[0].bufsize = sizeof(*mac_tbl_uc) - + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); + if (nvq_pairs <= 1) + return EINVAL; - specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; - specs[1].buf = mac_tbl_mc; - specs[1].bufsize = sizeof(*mac_tbl_mc) - + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); + vioif_ctrl_acquire(sc); - r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); - if (r != 0) - goto out; + mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); + specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; + specs[0].buf = mq; + specs[0].bufsize = sizeof(*mq); r = vioif_ctrl_send_command(sc, - VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, - specs, nspecs); - - vioif_ctrl_unload_cmdspec(sc, specs, nspecs); + VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, + specs, __arraycount(specs)); -out: vioif_ctrl_release(sc); return r; @@ -2350,110 +2512,53 @@ vioif_set_mac_addr(struct vioif_softc *s VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]); } r = 0; - } - - return r; -} - -static int -vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs) -{ - struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq; - struct vioif_ctrl_cmdspec specs[1]; - int r; - - if (!sc->sc_has_ctrl) - return ENOTSUP; - - if (nvq_pairs <= 1) - return EINVAL; - - vioif_ctrl_acquire(sc); - - mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs); - specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap; - specs[0].buf = mq; - specs[0].bufsize = sizeof(*mq); - - r = vioif_ctrl_send_command(sc, - VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, - specs, __arraycount(specs)); - - vioif_ctrl_release(sc); + } return r; } -/* ctrl vq interrupt; wake up the command issuer */ static int -vioif_ctrl_intr(void *arg) +vioif_set_rx_filter(struct vioif_softc *sc) { - struct vioif_ctrlqueue *ctrlq = arg; - struct virtqueue *vq = ctrlq->ctrlq_vq; - struct virtio_softc *vsc = vq->vq_owner; - int r, slot; - - if (virtio_vq_is_enqueued(vsc, vq) == false) - return 0; + /* filter already set in ctrlq->ctrlq_mac_tbl */ + struct virtio_softc *vsc = sc->sc_virtio; + struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc; + struct vioif_ctrl_cmdspec specs[2]; + int nspecs = __arraycount(specs); + int r; - r = virtio_dequeue(vsc, vq, &slot, NULL); - if (r == ENOENT) - return 0; - virtio_dequeue_commit(vsc, vq, slot); + mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc; + mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc; - mutex_enter(&ctrlq->ctrlq_wait_lock); - ctrlq->ctrlq_inuse = DONE; - cv_signal(&ctrlq->ctrlq_wait); - mutex_exit(&ctrlq->ctrlq_wait_lock); + if (!sc->sc_has_ctrl) + return ENOTSUP; - return 1; -} + vioif_ctrl_acquire(sc); -static int -vioif_ifflags(struct vioif_softc *sc) -{ - struct ifnet *ifp = &sc->sc_ethercom.ec_if; - bool onoff; - int r; + specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap; + specs[0].buf = mac_tbl_uc; + specs[0].bufsize = sizeof(*mac_tbl_uc) + + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries)); - if (!sc->sc_has_ctrl) { - /* no ctrl vq; always promisc and allmulti */ - ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI); - return 0; - } + specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap; + specs[1].buf = mac_tbl_mc; + specs[1].bufsize = sizeof(*mac_tbl_mc) + + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries)); - onoff = ifp->if_flags & IFF_ALLMULTI ? true : false; - r = vioif_set_allmulti(sc, onoff); - if (r != 0) { - log(LOG_WARNING, - "%s: couldn't %sable ALLMULTI\n", - ifp->if_xname, onoff ? "en" : "dis"); - if (onoff == false) { - ifp->if_flags |= IFF_ALLMULTI; - } - } + r = vioif_ctrl_load_cmdspec(sc, specs, nspecs); + if (r != 0) + goto out; - onoff = ifp->if_flags & IFF_PROMISC ? true : false; - r = vioif_set_promisc(sc, onoff); - if (r != 0) { - log(LOG_WARNING, - "%s: couldn't %sable PROMISC\n", - ifp->if_xname, onoff ? "en" : "dis"); - if (onoff == false) { - ifp->if_flags |= IFF_PROMISC; - } - } + r = vioif_ctrl_send_command(sc, + VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET, + specs, nspecs); - return 0; -} + vioif_ctrl_unload_cmdspec(sc, specs, nspecs); -static int -vioif_ifflags_cb(struct ethercom *ec) -{ - struct ifnet *ifp = &ec->ec_if; - struct vioif_softc *sc = ifp->if_softc; +out: + vioif_ctrl_release(sc); - return vioif_ifflags(sc); + return r; } /* @@ -2536,6 +2641,28 @@ set_ifflags: return r; } +/* + * VM configuration changes + */ +static int +vioif_config_change(struct virtio_softc *vsc) +{ + struct vioif_softc *sc = device_private(virtio_child(vsc)); + + softint_schedule(sc->sc_cfg_softint); + return 0; +} + +static void +vioif_cfg_softint(void *arg) +{ + struct vioif_softc *sc = arg; + struct ifnet *ifp = &sc->sc_ethercom.ec_if; + + vioif_update_link_status(sc); + vioif_start(ifp); +} + static int vioif_get_link_status(struct vioif_softc *sc) { @@ -2554,7 +2681,6 @@ vioif_get_link_status(struct vioif_softc return LINK_STATE_DOWN; } -/* change link status */ static void vioif_update_link_status(struct vioif_softc *sc) { @@ -2589,23 +2715,15 @@ done: mutex_exit(&sc->sc_lock); } -static int -vioif_config_change(struct virtio_softc *vsc) -{ - struct vioif_softc *sc = device_private(virtio_child(vsc)); - - softint_schedule(sc->sc_ctl_softint); - return 0; -} - static void -vioif_ctl_softint(void *arg) +vioif_workq_work(struct work *wk, void *context) { - struct vioif_softc *sc = arg; - struct ifnet *ifp = &sc->sc_ethercom.ec_if; + struct vioif_work *work; - vioif_update_link_status(sc); - vioif_start(ifp); + work = container_of(wk, struct vioif_work, cookie); + + atomic_store_relaxed(&work->added, 0); + work->func(work->arg); } static struct workqueue * @@ -2631,17 +2749,6 @@ vioif_workq_destroy(struct workqueue *wq } static void -vioif_workq_work(struct work *wk, void *context) -{ - struct vioif_work *work; - - work = container_of(wk, struct vioif_work, cookie); - - atomic_store_relaxed(&work->added, 0); - work->func(work->arg); -} - -static void vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg) { @@ -2670,125 +2777,6 @@ vioif_work_wait(struct workqueue *wq, st workqueue_wait(wq, &work->cookie); } -static int -vioif_setup_sysctl(struct vioif_softc *sc) -{ - const char *devname; - struct sysctllog **log; - const struct sysctlnode *rnode, *rxnode, *txnode; - int error; - - log = &sc->sc_sysctllog; - devname = device_xname(sc->sc_dev); - - error = sysctl_createv(log, 0, NULL, &rnode, - 0, CTLTYPE_NODE, devname, - SYSCTL_DESCR("virtio-net information and settings"), - NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue", - SYSCTL_DESCR("Use workqueue for packet processing"), - NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, &rxnode, - 0, CTLTYPE_NODE, "rx", - SYSCTL_DESCR("virtio-net information and settings for Rx"), - NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rxnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", - SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"), - NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rxnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", - SYSCTL_DESCR("max number of Rx packets to process for deferred processing"), - NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &rnode, &txnode, - 0, CTLTYPE_NODE, "tx", - SYSCTL_DESCR("virtio-net information and settings for Tx"), - NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &txnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit", - SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"), - NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL); - if (error) - goto out; - - error = sysctl_createv(log, 0, &txnode, NULL, - CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit", - SYSCTL_DESCR("max number of Tx packets to process for deferred processing"), - NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL); - -out: - if (error) - sysctl_teardown(log); - - return error; -} - -static void -vioif_setup_stats(struct vioif_softc *sc) -{ - struct vioif_netqueue *netq; - struct vioif_tx_context *txc; - struct vioif_rx_context *rxc; - size_t i, netq_num; - - netq_num = sc->sc_max_nvq_pairs * 2; - for (i = 0; i < netq_num; i++) { - netq = &sc->sc_netqs[i]; - evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC, - NULL, netq->netq_evgroup, "failed to load mbuf to DMA"); - evcnt_attach_dynamic(&netq->netq_enqueue_failed, - EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, - "virtqueue enqueue failed failed"); - - switch (VIOIF_NETQ_DIR(i)) { - case VIOIF_NETQ_RX: - rxc = netq->netq_ctx; - evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs, - EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, - "no receive buffer"); - break; - case VIOIF_NETQ_TX: - txc = netq->netq_ctx; - evcnt_attach_dynamic(&txc->txc_defrag_failed, - EVCNT_TYPE_MISC, NULL, netq->netq_evgroup, - "m_defrag() failed"); - break; - } - } - - evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC, - NULL, device_xname(sc->sc_dev), "control command dmamap load failed"); - evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC, - NULL, device_xname(sc->sc_dev), "control command failed"); -} - -static void -vioif_intr_barrier(void) -{ - - /* wait for finish all interrupt handler */ - xc_barrier(0); -} - MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio"); #ifdef _MODULE