Module Name: src
Committed By: hikaru
Date: Fri Nov 25 05:29:55 UTC 2016
Modified Files:
src/sys/arch/x86/pci: if_vmx.c if_vmxreg.h
Log Message:
Sync code with FreeBSD to support RSS
- Use MSI/MSI-X if it is available.
- Support TSO.
co-authored by k-nakahara
To generate a diff of this commit:
cvs rdiff -u -r1.7 -r1.8 src/sys/arch/x86/pci/if_vmx.c
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/x86/pci/if_vmxreg.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/pci/if_vmx.c
diff -u src/sys/arch/x86/pci/if_vmx.c:1.7 src/sys/arch/x86/pci/if_vmx.c:1.8
--- src/sys/arch/x86/pci/if_vmx.c:1.7 Fri Jun 10 13:27:13 2016
+++ src/sys/arch/x86/pci/if_vmx.c Fri Nov 25 05:29:54 2016
@@ -1,8 +1,9 @@
-/* $NetBSD: if_vmx.c,v 1.7 2016/06/10 13:27:13 ozaki-r Exp $ */
+/* $NetBSD: if_vmx.c,v 1.8 2016/11/25 05:29:54 hikaru Exp $ */
/* $OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $ */
/*
* Copyright (c) 2013 Tsubai Masanari
+ * Copyright (c) 2013 Bryan Venteicher <[email protected]>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
@@ -18,9 +19,12 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.7 2016/06/10 13:27:13 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.8 2016/11/25 05:29:54 hikaru Exp $");
#include <sys/param.h>
+#include <sys/cpu.h>
+#include <sys/kernel.h>
+#include <sys/kmem.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/mbuf.h>
@@ -35,6 +39,7 @@ __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1
#include <netinet/in_systm.h> /* for <netinet/ip.h> */
#include <netinet/in.h> /* for <netinet/ip.h> */
#include <netinet/ip.h> /* for struct ip */
+#include <netinet/ip6.h> /* for struct ip6_hdr */
#include <netinet/tcp.h> /* for struct tcphdr */
#include <netinet/udp.h> /* for struct udphdr */
@@ -44,137 +49,446 @@ __KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1
#include <arch/x86/pci/if_vmxreg.h>
-#define NRXQUEUE 1
-#define NTXQUEUE 1
+#define VMXNET3_DRIVER_VERSION 0x00010000
-#define NTXDESC 128 /* tx ring size */
-#define NTXSEGS 8 /* tx descriptors per packet */
-#define NRXDESC 128
-#define NTXCOMPDESC NTXDESC
-#define NRXCOMPDESC (NRXDESC * 2) /* ring1 + ring2 */
+/*
+ * Max descriptors per Tx packet. We must limit the size of the
+ * any TSO packets based on the number of segments.
+ */
+#define VMXNET3_TX_MAXSEGS 32
+#define VMXNET3_TX_MAXSIZE (VMXNET3_TX_MAXSEGS * MCLBYTES)
-#define VMXNET3_DRIVER_VERSION 0x00010000
+/*
+ * Maximum support Tx segments size. The length field in the
+ * Tx descriptor is 14 bits.
+ */
+#define VMXNET3_TX_MAXSEGSIZE (1 << 14)
+
+/*
+ * The maximum number of Rx segments we accept.
+ */
+#define VMXNET3_MAX_RX_SEGS 0 /* no segments */
+
+/*
+ * Predetermined size of the multicast MACs filter table. If the
+ * number of multicast addresses exceeds this size, then the
+ * ALL_MULTI mode is use instead.
+ */
+#define VMXNET3_MULTICAST_MAX 32
+
+/*
+ * Our Tx watchdog timeout.
+ */
+#define VMXNET3_WATCHDOG_TIMEOUT 5
+
+/*
+ * IP protocols that we can perform Tx checksum offloading of.
+ */
+#define VMXNET3_CSUM_OFFLOAD \
+ (M_CSUM_TCPv4 | M_CSUM_UDPv4)
+#define VMXNET3_CSUM_OFFLOAD_IPV6 \
+ (M_CSUM_TCPv6 | M_CSUM_UDPv6)
+
+#define VMXNET3_CSUM_ALL_OFFLOAD \
+ (VMXNET3_CSUM_OFFLOAD | VMXNET3_CSUM_OFFLOAD_IPV6 | M_CSUM_TSOv4 | M_CSUM_TSOv6)
+
+#define VMXNET3_RXRINGS_PERQ 2
+
+#define VMXNET3_CORE_LOCK(_sc) mutex_enter((_sc)->vmx_mtx)
+#define VMXNET3_CORE_UNLOCK(_sc) mutex_exit((_sc)->vmx_mtx)
+#define VMXNET3_CORE_LOCK_ASSERT(_sc) mutex_owned((_sc)->vmx_mtx)
+#define VMXNET3_CORE_LOCK_ASSERT_NOTOWNED(_sc) \
+ (!mutex_owned((_sc)->vmx_mtx))
+
+#define VMXNET3_RXQ_LOCK(_rxq) mutex_enter((_rxq)->vxrxq_mtx)
+#define VMXNET3_RXQ_UNLOCK(_rxq) mutex_exit((_rxq)->vxrxq_mtx)
+#define VMXNET3_RXQ_LOCK_ASSERT(_rxq) \
+ mutex_owned((_rxq)->vxrxq_mtx)
+#define VMXNET3_RXQ_LOCK_ASSERT_NOTOWNED(_rxq) \
+ (!mutex_owned((_rxq)->vxrxq_mtx))
+
+#define VMXNET3_TXQ_LOCK(_txq) mutex_enter((_txq)->vxtxq_mtx)
+#define VMXNET3_TXQ_UNLOCK(_txq) mutex_exit((_txq)->vxtxq_mtx)
+#define VMXNET3_TXQ_LOCK_ASSERT(_txq) \
+ mutex_owned((_txq)->vxtxq_mtx)
+#define VMXNET3_TXQ_LOCK_ASSERT_NOTOWNED(_txq) \
+ (!mutex_owned((_txq)->vxtxq_mtx))
+
+struct vmxnet3_dma_alloc {
+ bus_addr_t dma_paddr;
+ void *dma_vaddr;
+ bus_dmamap_t dma_map;
+ bus_size_t dma_size;
+ bus_dma_segment_t dma_segs[1];
+};
+
+struct vmxnet3_txbuf {
+ bus_dmamap_t vtxb_dmamap;
+ struct mbuf *vtxb_m;
+};
struct vmxnet3_txring {
- struct mbuf *m[NTXDESC];
- bus_dmamap_t dmap[NTXDESC];
- struct vmxnet3_txdesc *txd;
- u_int head;
- u_int next;
- uint8_t gen;
+ struct vmxnet3_txbuf *vxtxr_txbuf;
+ struct vmxnet3_txdesc *vxtxr_txd;
+ u_int vxtxr_head;
+ u_int vxtxr_next;
+ u_int vxtxr_ndesc;
+ int vxtxr_gen;
+ struct vmxnet3_dma_alloc vxtxr_dma;
+};
+
+struct vmxnet3_rxbuf {
+ bus_dmamap_t vrxb_dmamap;
+ struct mbuf *vrxb_m;
};
struct vmxnet3_rxring {
- struct mbuf *m[NRXDESC];
- bus_dmamap_t dmap[NRXDESC];
- struct vmxnet3_rxdesc *rxd;
- u_int fill;
- uint8_t gen;
- uint8_t rid;
+ struct vmxnet3_rxbuf *vxrxr_rxbuf;
+ struct vmxnet3_rxdesc *vxrxr_rxd;
+ u_int vxrxr_fill;
+ u_int vxrxr_ndesc;
+ int vxrxr_gen;
+ int vxrxr_rid;
+ struct vmxnet3_dma_alloc vxrxr_dma;
+ bus_dmamap_t vxrxr_spare_dmap;
};
struct vmxnet3_comp_ring {
union {
struct vmxnet3_txcompdesc *txcd;
struct vmxnet3_rxcompdesc *rxcd;
- };
- u_int next;
- uint8_t gen;
+ } vxcr_u;
+ u_int vxcr_next;
+ u_int vxcr_ndesc;
+ int vxcr_gen;
+ struct vmxnet3_dma_alloc vxcr_dma;
+};
+
+struct vmxnet3_txq_stats {
+ uint64_t vmtxs_opackets; /* if_opackets */
+ uint64_t vmtxs_obytes; /* if_obytes */
+ uint64_t vmtxs_omcasts; /* if_omcasts */
+ uint64_t vmtxs_csum;
+ uint64_t vmtxs_tso;
+ uint64_t vmtxs_full;
+ uint64_t vmtxs_offload_failed;
};
struct vmxnet3_txqueue {
- struct vmxnet3_txring cmd_ring;
- struct vmxnet3_comp_ring comp_ring;
- struct vmxnet3_txq_shared *ts;
+ kmutex_t *vxtxq_mtx;
+ struct vmxnet3_softc *vxtxq_sc;
+ int vxtxq_id;
+ int vxtxq_intr_idx;
+ int vxtxq_watchdog;
+ struct vmxnet3_txring vxtxq_cmd_ring;
+ struct vmxnet3_comp_ring vxtxq_comp_ring;
+ struct vmxnet3_txq_stats vxtxq_stats;
+ struct vmxnet3_txq_shared *vxtxq_ts;
+ char vxtxq_name[16];
+};
+
+struct vmxnet3_rxq_stats {
+ uint64_t vmrxs_ipackets; /* if_ipackets */
+ uint64_t vmrxs_ibytes; /* if_ibytes */
+ uint64_t vmrxs_iqdrops; /* if_iqdrops */
+ uint64_t vmrxs_ierrors; /* if_ierrors */
};
struct vmxnet3_rxqueue {
- struct vmxnet3_rxring cmd_ring[2];
- struct vmxnet3_comp_ring comp_ring;
- struct vmxnet3_rxq_shared *rs;
+ kmutex_t *vxrxq_mtx;
+ struct vmxnet3_softc *vxrxq_sc;
+ int vxrxq_id;
+ int vxrxq_intr_idx;
+ struct mbuf *vxrxq_mhead;
+ struct mbuf *vxrxq_mtail;
+ struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ];
+ struct vmxnet3_comp_ring vxrxq_comp_ring;
+ struct vmxnet3_rxq_stats vxrxq_stats;
+ struct vmxnet3_rxq_shared *vxrxq_rs;
+ char vxrxq_name[16];
+};
+
+struct vmxnet3_statistics {
+ uint32_t vmst_defragged;
+ uint32_t vmst_defrag_failed;
+ uint32_t vmst_mgetcl_failed;
+ uint32_t vmst_mbuf_load_failed;
};
struct vmxnet3_softc {
- device_t sc_dev;
- struct ethercom sc_ethercom;
- struct ifmedia sc_media;
-
- bus_space_tag_t sc_iot0;
- bus_space_tag_t sc_iot1;
- bus_space_handle_t sc_ioh0;
- bus_space_handle_t sc_ioh1;
- bus_dma_tag_t sc_dmat;
-
- struct vmxnet3_txqueue sc_txq[NTXQUEUE];
- struct vmxnet3_rxqueue sc_rxq[NRXQUEUE];
- struct vmxnet3_driver_shared *sc_ds;
- uint8_t *sc_mcast;
+ device_t vmx_dev;
+ struct ethercom vmx_ethercom;
+ struct ifmedia vmx_media;
+ struct vmxnet3_driver_shared *vmx_ds;
+ int vmx_flags;
+#define VMXNET3_FLAG_NO_MSIX (1 << 0)
+#define VMXNET3_FLAG_RSS (1 << 1)
+#define VMXNET3_FLAG_ATTACHED (1 << 2)
+
+ struct vmxnet3_txqueue *vmx_txq;
+ struct vmxnet3_rxqueue *vmx_rxq;
+
+ struct pci_attach_args *vmx_pa;
+
+ bus_space_tag_t vmx_iot0;
+ bus_space_tag_t vmx_iot1;
+ bus_space_handle_t vmx_ioh0;
+ bus_space_handle_t vmx_ioh1;
+ bus_size_t vmx_ios0;
+ bus_size_t vmx_ios1;
+ bus_dma_tag_t vmx_dmat;
+
+ int vmx_link_active;
+ int vmx_ntxqueues;
+ int vmx_nrxqueues;
+ int vmx_ntxdescs;
+ int vmx_nrxdescs;
+ int vmx_max_rxsegs;
+
+ struct vmxnet3_statistics vmx_stats;
+
+ int vmx_intr_type;
+ int vmx_intr_mask_mode;
+ int vmx_event_intr_idx;
+ int vmx_nintrs;
+ pci_intr_handle_t *vmx_intrs; /* legacy use vmx_intrs[0] */
+ void *vmx_ihs[VMXNET3_MAX_INTRS];
+
+ kmutex_t *vmx_mtx;
+
+ uint8_t *vmx_mcast;
+ void *vmx_qs;
+ struct vmxnet3_rss_shared *vmx_rss;
+ callout_t vmx_tick;
+ struct vmxnet3_dma_alloc vmx_ds_dma;
+ struct vmxnet3_dma_alloc vmx_qs_dma;
+ struct vmxnet3_dma_alloc vmx_mcast_dma;
+ struct vmxnet3_dma_alloc vmx_rss_dma;
+ int vmx_max_ntxqueues;
+ int vmx_max_nrxqueues;
+ uint8_t vmx_lladdr[ETHER_ADDR_LEN];
};
#define VMXNET3_STAT
#ifdef VMXNET3_STAT
struct {
- u_int ntxdesc;
- u_int nrxdesc;
u_int txhead;
u_int txdone;
u_int maxtxlen;
u_int rxdone;
u_int rxfill;
u_int intr;
-} vmxstat = {
- .ntxdesc = NTXDESC,
- .nrxdesc = NRXDESC
-};
+} vmxstat;
#endif
+typedef enum {
+ VMXNET3_BARRIER_RD,
+ VMXNET3_BARRIER_WR,
+ VMXNET3_BARRIER_RDWR,
+} vmxnet3_barrier_t;
+
#define JUMBO_LEN (MCLBYTES - ETHER_ALIGN) /* XXX */
#define DMAADDR(map) ((map)->dm_segs[0].ds_addr)
-#define READ_BAR0(sc, reg) bus_space_read_4((sc)->sc_iot0, (sc)->sc_ioh0, reg)
-#define READ_BAR1(sc, reg) bus_space_read_4((sc)->sc_iot1, (sc)->sc_ioh1, reg)
-#define WRITE_BAR0(sc, reg, val) \
- bus_space_write_4((sc)->sc_iot0, (sc)->sc_ioh0, reg, val)
-#define WRITE_BAR1(sc, reg, val) \
- bus_space_write_4((sc)->sc_iot1, (sc)->sc_ioh1, reg, val)
-#define WRITE_CMD(sc, cmd) WRITE_BAR1(sc, VMXNET3_BAR1_CMD, cmd)
#define vtophys(va) 0 /* XXX ok? */
int vmxnet3_match(device_t, cfdata_t, void *);
void vmxnet3_attach(device_t, device_t, void *);
-int vmxnet3_dma_init(struct vmxnet3_softc *);
-int vmxnet3_alloc_txring(struct vmxnet3_softc *, int);
-int vmxnet3_alloc_rxring(struct vmxnet3_softc *, int);
-void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
-void vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
+int vmxnet3_detach(device_t, int);
+
+int vmxnet3_alloc_pci_resources(struct vmxnet3_softc *);
+void vmxnet3_free_pci_resources(struct vmxnet3_softc *);
+int vmxnet3_check_version(struct vmxnet3_softc *);
+void vmxnet3_check_multiqueue(struct vmxnet3_softc *);
+
+int vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *);
+int vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *);
+int vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *);
+int vmxnet3_alloc_interrupts(struct vmxnet3_softc *);
+void vmxnet3_free_interrupts(struct vmxnet3_softc *);
+
+int vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *);
+int vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *);
+int vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *);
+void vmxnet3_set_interrupt_idx(struct vmxnet3_softc *);
+int vmxnet3_setup_interrupts(struct vmxnet3_softc *);
+
+int vmxnet3_init_rxq(struct vmxnet3_softc *, int);
+int vmxnet3_init_txq(struct vmxnet3_softc *, int);
+int vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *);
+void vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *);
+void vmxnet3_destroy_txq(struct vmxnet3_txqueue *);
+void vmxnet3_free_rxtx_queues(struct vmxnet3_softc *);
+
+int vmxnet3_alloc_shared_data(struct vmxnet3_softc *);
+void vmxnet3_free_shared_data(struct vmxnet3_softc *);
+int vmxnet3_alloc_txq_data(struct vmxnet3_softc *);
+void vmxnet3_free_txq_data(struct vmxnet3_softc *);
+int vmxnet3_alloc_rxq_data(struct vmxnet3_softc *);
+void vmxnet3_free_rxq_data(struct vmxnet3_softc *);
+int vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
+void vmxnet3_free_queue_data(struct vmxnet3_softc *);
+int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
+void vmxnet3_free_mcast_table(struct vmxnet3_softc *);
+void vmxnet3_init_shared_data(struct vmxnet3_softc *);
+void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
+void vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
+int vmxnet3_alloc_data(struct vmxnet3_softc *);
+void vmxnet3_free_data(struct vmxnet3_softc *);
+int vmxnet3_setup_interface(struct vmxnet3_softc *);
+
+void vmxnet3_evintr(struct vmxnet3_softc *);
+void vmxnet3_txq_eof(struct vmxnet3_txqueue *);
+int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
+void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
+ struct vmxnet3_rxring *, int);
+void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *);
+void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
+void vmxnet3_rxq_input(struct vmxnet3_rxqueue *,
+ struct vmxnet3_rxcompdesc *, struct mbuf *);
+void vmxnet3_rxq_eof(struct vmxnet3_rxqueue *);
+int vmxnet3_legacy_intr(void *);
+int vmxnet3_txq_intr(void *);
+int vmxnet3_rxq_intr(void *);
+int vmxnet3_event_intr(void *);
+
void vmxnet3_txstop(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
void vmxnet3_rxstop(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
-void vmxnet3_link_state(struct vmxnet3_softc *);
-void vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
-void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
-int vmxnet3_intr(void *);
-void vmxnet3_evintr(struct vmxnet3_softc *);
-void vmxnet3_txintr(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
-void vmxnet3_rxintr(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
-void vmxnet3_iff(struct vmxnet3_softc *);
-int vmxnet3_ifflags_cb(struct ethercom *);
+void vmxnet3_stop_locked(struct vmxnet3_softc *);
+void vmxnet3_stop_rendezvous(struct vmxnet3_softc *);
+void vmxnet3_stop(struct ifnet *, int);
+
+void vmxnet3_txinit(struct vmxnet3_softc *, struct vmxnet3_txqueue *);
+int vmxnet3_rxinit(struct vmxnet3_softc *, struct vmxnet3_rxqueue *);
+int vmxnet3_reinit_queues(struct vmxnet3_softc *);
+int vmxnet3_enable_device(struct vmxnet3_softc *);
+void vmxnet3_reinit_rxfilters(struct vmxnet3_softc *);
+int vmxnet3_reinit(struct vmxnet3_softc *);
+
void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
-int vmxnet3_getbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
-void vmxnet3_stop(struct ifnet *, int disable);
-void vmxnet3_reset(struct ifnet *);
+int vmxnet3_init_locked(struct vmxnet3_softc *);
int vmxnet3_init(struct ifnet *);
-int vmxnet3_ioctl(struct ifnet *, u_long, void *);
-int vmxnet3_change_mtu(struct vmxnet3_softc *, int);
+
+int vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *, struct mbuf *, int *, int *);
+int vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *, struct mbuf **, bus_dmamap_t);
+void vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *, bus_dmamap_t);
+int vmxnet3_txq_encap(struct vmxnet3_txqueue *, struct mbuf **);
+void vmxnet3_start_locked(struct ifnet *);
void vmxnet3_start(struct ifnet *);
-int vmxnet3_load_mbuf(struct vmxnet3_softc *, struct mbuf *);
-void vmxnet3_watchdog(struct ifnet *);
+
+void vmxnet3_set_rxfilter(struct vmxnet3_softc *);
+int vmxnet3_change_mtu(struct vmxnet3_softc *, int);
+int vmxnet3_ioctl(struct ifnet *, u_long, void *);
+int vmxnet3_ifflags_cb(struct ethercom *);
+
+int vmxnet3_watchdog(struct vmxnet3_txqueue *);
+void vmxnet3_refresh_host_stats(struct vmxnet3_softc *);
+void vmxnet3_tick(void *);
+void vmxnet3_link_status(struct vmxnet3_softc *);
void vmxnet3_media_status(struct ifnet *, struct ifmediareq *);
int vmxnet3_media_change(struct ifnet *);
-void *vmxnet3_dma_allocmem(struct vmxnet3_softc *, u_int, u_int, bus_addr_t *);
+void vmxnet3_set_lladdr(struct vmxnet3_softc *);
+void vmxnet3_get_lladdr(struct vmxnet3_softc *);
+
+void vmxnet3_enable_all_intrs(struct vmxnet3_softc *);
+void vmxnet3_disable_all_intrs(struct vmxnet3_softc *);
+
+int vmxnet3_dma_malloc(struct vmxnet3_softc *, bus_size_t, bus_size_t,
+ struct vmxnet3_dma_alloc *);
+void vmxnet3_dma_free(struct vmxnet3_softc *, struct vmxnet3_dma_alloc *);
CFATTACH_DECL3_NEW(vmx, sizeof(struct vmxnet3_softc),
- vmxnet3_match, vmxnet3_attach, NULL, NULL, NULL, NULL, 0);
+ vmxnet3_match, vmxnet3_attach, vmxnet3_detach, NULL, NULL, NULL, 0);
+
+static inline void
+vmxnet3_write_bar0(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
+{
+
+ bus_space_write_4(sc->vmx_iot0, sc->vmx_ioh0, r, v);
+}
+
+static inline uint32_t
+vmxnet3_read_bar1(struct vmxnet3_softc *sc, bus_size_t r)
+{
+
+ return (bus_space_read_4(sc->vmx_iot1, sc->vmx_ioh1, r));
+}
+
+static inline void
+vmxnet3_write_bar1(struct vmxnet3_softc *sc, bus_size_t r, uint32_t v)
+{
+
+ bus_space_write_4(sc->vmx_iot1, sc->vmx_ioh1, r, v);
+}
+
+static inline void
+vmxnet3_write_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
+{
+
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_CMD, cmd);
+}
+
+static inline uint32_t
+vmxnet3_read_cmd(struct vmxnet3_softc *sc, uint32_t cmd)
+{
+
+ vmxnet3_write_cmd(sc, cmd);
+ return (vmxnet3_read_bar1(sc, VMXNET3_BAR1_CMD));
+}
+
+static inline void
+vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
+{
+ vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 0);
+}
+
+static inline void
+vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
+{
+ vmxnet3_write_bar0(sc, VMXNET3_BAR0_IMASK(irq), 1);
+}
+
+static inline void
+vmxnet3_rxr_increment_fill(struct vmxnet3_rxring *rxr)
+{
+
+ if (++rxr->vxrxr_fill == rxr->vxrxr_ndesc) {
+ rxr->vxrxr_fill = 0;
+ rxr->vxrxr_gen ^= 1;
+ }
+}
+
+static inline int
+vmxnet3_txring_avail(struct vmxnet3_txring *txr)
+{
+ int avail = txr->vxtxr_next - txr->vxtxr_head - 1;
+ return (avail < 0 ? txr->vxtxr_ndesc + avail : avail);
+}
+
+/*
+ * Since this is a purely paravirtualized device, we do not have
+ * to worry about DMA coherency. But at times, we must make sure
+ * both the compiler and CPU do not reorder memory operations.
+ */
+static inline void
+vmxnet3_barrier(struct vmxnet3_softc *sc, vmxnet3_barrier_t type)
+{
+
+ switch (type) {
+ case VMXNET3_BARRIER_RD:
+ membar_consumer();
+ break;
+ case VMXNET3_BARRIER_WR:
+ membar_producer();
+ break;
+ case VMXNET3_BARRIER_RDWR:
+ membar_sync();
+ break;
+ default:
+ panic("%s: bad barrier type %d", __func__, type);
+ }
+}
int
vmxnet3_match(device_t parent, cfdata_t match, void *aux)
@@ -193,617 +507,2297 @@ vmxnet3_attach(device_t parent, device_t
{
struct vmxnet3_softc *sc = device_private(self);
struct pci_attach_args *pa = aux;
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- pci_intr_handle_t ih;
- const char *intrstr;
- void *vih;
- u_int memtype, ver, macl, mach;
pcireg_t preg;
- u_char enaddr[ETHER_ADDR_LEN];
- char intrbuf[PCI_INTRSTR_LEN];
+ int error;
- sc->sc_dev = self;
+ sc->vmx_dev = self;
+ sc->vmx_pa = pa;
+ if (pci_dma64_available(pa))
+ sc->vmx_dmat = pa->pa_dmat64;
+ else
+ sc->vmx_dmat = pa->pa_dmat;
pci_aprint_devinfo_fancy(pa, "Ethernet controller", "vmxnet3", 1);
- memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x10);
- if (pci_mapreg_map(pa, 0x10, memtype, 0, &sc->sc_iot0, &sc->sc_ioh0,
- NULL, NULL)) {
- aprint_error_dev(sc->sc_dev, "failed to map BAR0\n");
- return;
- }
- memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, 0x14);
- if (pci_mapreg_map(pa, 0x14, memtype, 0, &sc->sc_iot1, &sc->sc_ioh1,
- NULL, NULL)) {
- aprint_error_dev(sc->sc_dev, "failed to map BAR1\n");
- return;
- }
-
- ver = READ_BAR1(sc, VMXNET3_BAR1_VRRS);
- if ((ver & 0x1) == 0) {
- aprint_error_dev(sc->sc_dev,
- "unsupported hardware version 0x%x\n", ver);
- return;
- }
- WRITE_BAR1(sc, VMXNET3_BAR1_VRRS, 1);
-
- ver = READ_BAR1(sc, VMXNET3_BAR1_UVRS);
- if ((ver & 0x1) == 0) {
- aprint_error_dev(sc->sc_dev,
- "incompatiable UPT version 0x%x\n", ver);
- return;
- }
- WRITE_BAR1(sc, VMXNET3_BAR1_UVRS, 1);
-
preg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
preg |= PCI_COMMAND_MASTER_ENABLE;
pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, preg);
- if (pci_dma64_available(pa))
- sc->sc_dmat = pa->pa_dmat64;
- else
- sc->sc_dmat = pa->pa_dmat;
- if (vmxnet3_dma_init(sc)) {
- aprint_error_dev(sc->sc_dev, "failed to setup DMA\n");
+ sc->vmx_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
+ callout_init(&sc->vmx_tick, CALLOUT_MPSAFE);
+
+ sc->vmx_max_ntxqueues = ncpu;
+ sc->vmx_max_nrxqueues = ncpu;
+ sc->vmx_ntxdescs = 512;
+ sc->vmx_nrxdescs = 256;
+ sc->vmx_max_rxsegs = VMXNET3_MAX_RX_SEGS;
+
+ error = vmxnet3_alloc_pci_resources(sc);
+ if (error)
return;
- }
- if (pci_intr_map(pa, &ih)) {
- aprint_error_dev(sc->sc_dev, "failed to map interrupt\n");
+ error = vmxnet3_check_version(sc);
+ if (error)
return;
- }
- intrstr = pci_intr_string(pa->pa_pc, ih, intrbuf, sizeof(intrbuf));
- vih = pci_intr_establish(pa->pa_pc, ih, IPL_NET, vmxnet3_intr, sc);
- if (vih == NULL) {
- aprint_error_dev(sc->sc_dev,
- "unable to establish interrupt%s%s\n",
- intrstr ? " at " : "", intrstr ? intrstr : "");
+
+ error = vmxnet3_alloc_rxtx_queues(sc);
+ if (error)
return;
- }
- aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
- WRITE_CMD(sc, VMXNET3_CMD_GET_MACL);
- macl = READ_BAR1(sc, VMXNET3_BAR1_CMD);
- enaddr[0] = macl;
- enaddr[1] = macl >> 8;
- enaddr[2] = macl >> 16;
- enaddr[3] = macl >> 24;
- WRITE_CMD(sc, VMXNET3_CMD_GET_MACH);
- mach = READ_BAR1(sc, VMXNET3_BAR1_CMD);
- enaddr[4] = mach;
- enaddr[5] = mach >> 8;
-
- WRITE_BAR1(sc, VMXNET3_BAR1_MACL, macl);
- WRITE_BAR1(sc, VMXNET3_BAR1_MACH, mach);
- aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
- ether_sprintf(enaddr));
+ error = vmxnet3_alloc_interrupts(sc);
+ if (error)
+ return;
- strlcpy(ifp->if_xname, device_xname(sc->sc_dev), IFNAMSIZ);
- ifp->if_softc = sc;
- ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
- ifp->if_ioctl = vmxnet3_ioctl;
- ifp->if_start = vmxnet3_start;
- ifp->if_watchdog = vmxnet3_watchdog;
- ifp->if_init = vmxnet3_init;
- ifp->if_stop = vmxnet3_stop;
- sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
- if (sc->sc_ds->upt_features & UPT1_F_CSUM)
- sc->sc_ethercom.ec_if.if_capabilities |=
- IFCAP_CSUM_IPv4_Rx |
- IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
- IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx;
- if (sc->sc_ds->upt_features & UPT1_F_VLAN)
- sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING;
+ vmxnet3_check_multiqueue(sc);
- IFQ_SET_MAXLEN(&ifp->if_snd, NTXDESC);
- IFQ_SET_READY(&ifp->if_snd);
+ error = vmxnet3_alloc_data(sc);
+ if (error)
+ return;
- ifmedia_init(&sc->sc_media, IFM_IMASK, vmxnet3_media_change,
- vmxnet3_media_status);
- ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_AUTO, 0, NULL);
- ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T|IFM_FDX, 0, NULL);
- ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_10G_T, 0, NULL);
- ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T|IFM_FDX, 0, NULL);
- ifmedia_add(&sc->sc_media, IFM_ETHER|IFM_1000_T, 0, NULL);
- ifmedia_set(&sc->sc_media, IFM_ETHER|IFM_AUTO);
+ error = vmxnet3_setup_interface(sc);
+ if (error)
+ return;
- if_attach(ifp);
- ether_ifattach(ifp, enaddr);
- ether_set_ifflags_cb(&sc->sc_ethercom, vmxnet3_ifflags_cb);
- vmxnet3_link_state(sc);
+ error = vmxnet3_setup_interrupts(sc);
+ if (error)
+ return;
+
+ sc->vmx_flags |= VMXNET3_FLAG_ATTACHED;
}
int
-vmxnet3_dma_init(struct vmxnet3_softc *sc)
+vmxnet3_detach(device_t self, int flags)
{
- struct vmxnet3_driver_shared *ds;
- struct vmxnet3_txq_shared *ts;
- struct vmxnet3_rxq_shared *rs;
- bus_addr_t ds_pa, qs_pa, mcast_pa;
- int i, queue, qs_len;
- u_int major, minor, release_code, rev;
-
- qs_len = NTXQUEUE * sizeof *ts + NRXQUEUE * sizeof *rs;
- ts = vmxnet3_dma_allocmem(sc, qs_len, VMXNET3_DMADESC_ALIGN, &qs_pa);
- if (ts == NULL)
- return -1;
- for (queue = 0; queue < NTXQUEUE; queue++)
- sc->sc_txq[queue].ts = ts++;
- rs = (void *)ts;
- for (queue = 0; queue < NRXQUEUE; queue++)
- sc->sc_rxq[queue].rs = rs++;
-
- for (queue = 0; queue < NTXQUEUE; queue++)
- if (vmxnet3_alloc_txring(sc, queue))
- return -1;
- for (queue = 0; queue < NRXQUEUE; queue++)
- if (vmxnet3_alloc_rxring(sc, queue))
- return -1;
+ struct vmxnet3_softc *sc;
+ struct ifnet *ifp;
- sc->sc_mcast = vmxnet3_dma_allocmem(sc, 682 * ETHER_ADDR_LEN, 32, &mcast_pa);
- if (sc->sc_mcast == NULL)
- return -1;
+ sc = device_private(self);
+ ifp = &sc->vmx_ethercom.ec_if;
- ds = vmxnet3_dma_allocmem(sc, sizeof *sc->sc_ds, 8, &ds_pa);
- if (ds == NULL)
- return -1;
- sc->sc_ds = ds;
- ds->magic = VMXNET3_REV1_MAGIC;
- ds->version = VMXNET3_DRIVER_VERSION;
+ if (sc->vmx_flags & VMXNET3_FLAG_ATTACHED) {
+ VMXNET3_CORE_LOCK(sc);
+ vmxnet3_stop_locked(sc);
+ callout_halt(&sc->vmx_tick, sc->vmx_mtx);
+ VMXNET3_CORE_UNLOCK(sc);
- /*
- * XXX FreeBSD version uses following values:
- * (Does the device behavior depend on them?)
- *
- * major = __FreeBSD_version / 100000;
- * minor = (__FreeBSD_version / 1000) % 100;
- * release_code = (__FreeBSD_version / 100) % 10;
- * rev = __FreeBSD_version % 100;
- */
- major = 0;
- minor = 0;
- release_code = 0;
- rev = 0;
-#ifdef __LP64__
- ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6
- | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_64BIT;
-#else
- ds->guest = release_code << 30 | rev << 22 | major << 14 | minor << 6
- | VMXNET3_GOS_FREEBSD | VMXNET3_GOS_32BIT;
-#endif
- ds->vmxnet3_revision = 1;
- ds->upt_version = 1;
- ds->upt_features = UPT1_F_CSUM | UPT1_F_VLAN;
- ds->driver_data = vtophys(sc);
- ds->driver_data_len = sizeof(struct vmxnet3_softc);
- ds->queue_shared = qs_pa;
- ds->queue_shared_len = qs_len;
- ds->mtu = ETHERMTU;
- ds->ntxqueue = NTXQUEUE;
- ds->nrxqueue = NRXQUEUE;
- ds->mcast_table = mcast_pa;
- ds->automask = 1;
- ds->nintr = VMXNET3_NINTR;
- ds->evintr = 0;
- ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
- ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
- WRITE_BAR1(sc, VMXNET3_BAR1_DSL, ds_pa);
- WRITE_BAR1(sc, VMXNET3_BAR1_DSH, (uint64_t)ds_pa >> 32);
- return 0;
-}
+ ifmedia_delete_instance(&sc->vmx_media, IFM_INST_ANY);
-int
-vmxnet3_alloc_txring(struct vmxnet3_softc *sc, int queue)
-{
- struct vmxnet3_txqueue *tq = &sc->sc_txq[queue];
- struct vmxnet3_txq_shared *ts;
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
- bus_addr_t pa, comp_pa;
- int idx;
+ ether_ifdetach(ifp);
+ if_detach(ifp);
+ }
- ring->txd = vmxnet3_dma_allocmem(sc, NTXDESC * sizeof ring->txd[0], 512, &pa);
- if (ring->txd == NULL)
- return -1;
- comp_ring->txcd = vmxnet3_dma_allocmem(sc,
- NTXCOMPDESC * sizeof comp_ring->txcd[0], 512, &comp_pa);
- if (comp_ring->txcd == NULL)
- return -1;
+ vmxnet3_free_interrupts(sc);
- for (idx = 0; idx < NTXDESC; idx++) {
- if (bus_dmamap_create(sc->sc_dmat, JUMBO_LEN, NTXSEGS,
- JUMBO_LEN, 0, BUS_DMA_NOWAIT, &ring->dmap[idx]))
- return -1;
- }
-
- ts = tq->ts;
- memset(ts, 0, sizeof *ts);
- ts->npending = 0;
- ts->intr_threshold = 1;
- ts->cmd_ring = pa;
- ts->cmd_ring_len = NTXDESC;
- ts->comp_ring = comp_pa;
- ts->comp_ring_len = NTXCOMPDESC;
- ts->driver_data = vtophys(tq);
- ts->driver_data_len = sizeof *tq;
- ts->intr_idx = 0;
- ts->stopped = 1;
- ts->error = 0;
- return 0;
+ vmxnet3_free_data(sc);
+ vmxnet3_free_pci_resources(sc);
+ vmxnet3_free_rxtx_queues(sc);
+
+ if (sc->vmx_mtx)
+ mutex_obj_free(sc->vmx_mtx);
+
+ return (0);
}
int
-vmxnet3_alloc_rxring(struct vmxnet3_softc *sc, int queue)
+vmxnet3_alloc_pci_resources(struct vmxnet3_softc *sc)
{
- struct vmxnet3_rxqueue *rq = &sc->sc_rxq[queue];
- struct vmxnet3_rxq_shared *rs;
- struct vmxnet3_rxring *ring;
- struct vmxnet3_comp_ring *comp_ring;
- bus_addr_t pa[2], comp_pa;
- int i, idx;
-
- for (i = 0; i < 2; i++) {
- ring = &rq->cmd_ring[i];
- ring->rxd = vmxnet3_dma_allocmem(sc, NRXDESC * sizeof ring->rxd[0],
- 512, &pa[i]);
- if (ring->rxd == NULL)
- return -1;
- }
- comp_ring = &rq->comp_ring;
- comp_ring->rxcd = vmxnet3_dma_allocmem(sc,
- NRXCOMPDESC * sizeof comp_ring->rxcd[0], 512, &comp_pa);
- if (comp_ring->rxcd == NULL)
- return -1;
+ struct pci_attach_args *pa = sc->vmx_pa;
+ pcireg_t memtype;
- for (i = 0; i < 2; i++) {
- ring = &rq->cmd_ring[i];
- ring->rid = i;
- for (idx = 0; idx < NRXDESC; idx++) {
- if (bus_dmamap_create(sc->sc_dmat, JUMBO_LEN, 1,
- JUMBO_LEN, 0, BUS_DMA_NOWAIT, &ring->dmap[idx]))
- return -1;
- }
+ memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(0));
+ if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->vmx_iot0, &sc->vmx_ioh0,
+ NULL, &sc->vmx_ios0)) {
+ aprint_error_dev(sc->vmx_dev, "failed to map BAR0\n");
+ return (ENXIO);
+ }
+ memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_BAR(1));
+ if (pci_mapreg_map(pa, PCI_BAR(1), memtype, 0, &sc->vmx_iot1, &sc->vmx_ioh1,
+ NULL, &sc->vmx_ios1)) {
+ aprint_error_dev(sc->vmx_dev, "failed to map BAR1\n");
+ return (ENXIO);
+ }
+
+ if (!pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, NULL, NULL)) {
+ sc->vmx_flags |= VMXNET3_FLAG_NO_MSIX;
+ return (0);
}
- rs = rq->rs;
- memset(rs, 0, sizeof *rs);
- rs->cmd_ring[0] = pa[0];
- rs->cmd_ring[1] = pa[1];
- rs->cmd_ring_len[0] = NRXDESC;
- rs->cmd_ring_len[1] = NRXDESC;
- rs->comp_ring = comp_pa;
- rs->comp_ring_len = NRXCOMPDESC;
- rs->driver_data = vtophys(rq);
- rs->driver_data_len = sizeof *rq;
- rs->intr_idx = 0;
- rs->stopped = 1;
- rs->error = 0;
- return 0;
+ return (0);
}
void
-vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq)
+vmxnet3_free_pci_resources(struct vmxnet3_softc *sc)
{
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
-
- ring->head = ring->next = 0;
- ring->gen = 1;
- comp_ring->next = 0;
- comp_ring->gen = 1;
- memset(ring->txd, 0, NTXDESC * sizeof ring->txd[0]);
- memset(comp_ring->txcd, 0, NTXCOMPDESC * sizeof comp_ring->txcd[0]);
-}
-void
-vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq)
-{
- struct vmxnet3_rxring *ring;
- struct vmxnet3_comp_ring *comp_ring;
- int i, idx;
+ if (sc->vmx_ios0) {
+ bus_space_unmap(sc->vmx_iot0, sc->vmx_ioh0, sc->vmx_ios0);
+ sc->vmx_ios0 = 0;
+ }
- for (i = 0; i < 2; i++) {
- ring = &rq->cmd_ring[i];
- ring->fill = 0;
- ring->gen = 1;
- memset(ring->rxd, 0, NRXDESC * sizeof ring->rxd[0]);
- for (idx = 0; idx < NRXDESC; idx++) {
- if (vmxnet3_getbuf(sc, ring))
- break;
- }
+ if (sc->vmx_ios1) {
+ bus_space_unmap(sc->vmx_iot1, sc->vmx_ioh1, sc->vmx_ios1);
+ sc->vmx_ios1 = 0;
}
- comp_ring = &rq->comp_ring;
- comp_ring->next = 0;
- comp_ring->gen = 1;
- memset(comp_ring->rxcd, 0, NRXCOMPDESC * sizeof comp_ring->rxcd[0]);
}
-void
-vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq)
+int
+vmxnet3_check_version(struct vmxnet3_softc *sc)
{
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- int idx;
+ u_int ver;
- for (idx = 0; idx < NTXDESC; idx++) {
- if (ring->m[idx]) {
- bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]);
- m_freem(ring->m[idx]);
- ring->m[idx] = NULL;
- }
+ ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_VRRS);
+ if ((ver & 0x1) == 0) {
+ aprint_error_dev(sc->vmx_dev,
+ "unsupported hardware version 0x%x\n", ver);
+ return (ENOTSUP);
}
-}
-
-void
-vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq)
-{
- struct vmxnet3_rxring *ring;
- int i, idx;
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_VRRS, 1);
- for (i = 0; i < 2; i++) {
- ring = &rq->cmd_ring[i];
- for (idx = 0; idx < NRXDESC; idx++) {
- if (ring->m[idx]) {
- m_freem(ring->m[idx]);
- ring->m[idx] = NULL;
- }
- }
+ ver = vmxnet3_read_bar1(sc, VMXNET3_BAR1_UVRS);
+ if ((ver & 0x1) == 0) {
+ aprint_error_dev(sc->vmx_dev,
+ "incompatiable UPT version 0x%x\n", ver);
+ return (ENOTSUP);
}
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_UVRS, 1);
+
+ return (0);
}
void
-vmxnet3_link_state(struct vmxnet3_softc *sc)
+vmxnet3_check_multiqueue(struct vmxnet3_softc *sc)
{
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- u_int x, link, speed;
- WRITE_CMD(sc, VMXNET3_CMD_GET_LINK);
- x = READ_BAR1(sc, VMXNET3_BAR1_CMD);
- speed = x >> 16;
- if (x & 1) {
- ifp->if_baudrate = IF_Mbps(speed);
- link = LINK_STATE_UP;
- } else
- link = LINK_STATE_DOWN;
+ if (sc->vmx_intr_type != VMXNET3_IT_MSIX)
+ goto out;
- if_link_state_change(ifp, link);
-}
+ /* Just use the maximum configured for now. */
+ sc->vmx_nrxqueues = sc->vmx_max_nrxqueues;
+ sc->vmx_ntxqueues = sc->vmx_max_ntxqueues;
-static inline void
-vmxnet3_enable_intr(struct vmxnet3_softc *sc, int irq)
-{
- WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 0);
+ if (sc->vmx_nrxqueues > 1)
+ sc->vmx_flags |= VMXNET3_FLAG_RSS;
+
+ return;
+
+out:
+ sc->vmx_ntxqueues = 1;
+ sc->vmx_nrxqueues = 1;
}
-static inline void
-vmxnet3_disable_intr(struct vmxnet3_softc *sc, int irq)
+int
+vmxnet3_alloc_msix_interrupts(struct vmxnet3_softc *sc)
{
- WRITE_BAR0(sc, VMXNET3_BAR0_IMASK(irq), 1);
+ int required;
+ struct pci_attach_args *pa = sc->vmx_pa;
+
+ if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX)
+ return (1);
+
+ /* Allocate an additional vector for the events interrupt. */
+ required = sc->vmx_max_nrxqueues + sc->vmx_max_ntxqueues + 1;
+
+ if (pci_msix_count(pa->pa_pc, pa->pa_tag) < required)
+ return (1);
+
+ if (pci_msix_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
+ sc->vmx_nintrs = required;
+ return (0);
+ }
+
+ return (1);
}
-void
-vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
+int
+vmxnet3_alloc_msi_interrupts(struct vmxnet3_softc *sc)
{
- int i;
+ int nmsi, required;
+ struct pci_attach_args *pa = sc->vmx_pa;
- sc->sc_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
- vmxnet3_enable_intr(sc, i);
+ required = 1;
+
+ nmsi = pci_msi_count(pa->pa_pc, pa->pa_tag);
+ if (nmsi < required)
+ return (1);
+
+ if (pci_msi_alloc_exact(pa, &sc->vmx_intrs, required) == 0) {
+ sc->vmx_nintrs = required;
+ return (0);
+ }
+
+ return (1);
}
-void
-vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
+int
+vmxnet3_alloc_legacy_interrupts(struct vmxnet3_softc *sc)
{
- int i;
- sc->sc_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
- for (i = 0; i < VMXNET3_NINTR; i++)
- vmxnet3_disable_intr(sc, i);
+ if (pci_intx_alloc(sc->vmx_pa, &sc->vmx_intrs) == 0) {
+ sc->vmx_nintrs = 1;
+ return (0);
+ }
+
+ return (1);
}
int
-vmxnet3_intr(void *arg)
+vmxnet3_alloc_interrupts(struct vmxnet3_softc *sc)
{
- struct vmxnet3_softc *sc = arg;
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
+ u_int config;
+ int error;
+
+ config = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_INTRCFG);
+
+ sc->vmx_intr_type = config & 0x03;
+ sc->vmx_intr_mask_mode = (config >> 2) & 0x03;
+
+ switch (sc->vmx_intr_type) {
+ case VMXNET3_IT_AUTO:
+ sc->vmx_intr_type = VMXNET3_IT_MSIX;
+ /* FALLTHROUGH */
+ case VMXNET3_IT_MSIX:
+ error = vmxnet3_alloc_msix_interrupts(sc);
+ if (error == 0)
+ break;
+ sc->vmx_intr_type = VMXNET3_IT_MSI;
+ /* FALLTHROUGH */
+ case VMXNET3_IT_MSI:
+ error = vmxnet3_alloc_msi_interrupts(sc);
+ if (error == 0)
+ break;
+ sc->vmx_intr_type = VMXNET3_IT_LEGACY;
+ /* FALLTHROUGH */
+ case VMXNET3_IT_LEGACY:
+ error = vmxnet3_alloc_legacy_interrupts(sc);
+ if (error == 0)
+ break;
+ /* FALLTHROUGH */
+ default:
+ sc->vmx_intr_type = -1;
+ aprint_error_dev(sc->vmx_dev, "cannot allocate any interrupt resources\n");
+ return (ENXIO);
+ }
+
+ return (error);
+}
+
+void
+vmxnet3_free_interrupts(struct vmxnet3_softc *sc)
+{
+ pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
+ int i;
+
+ for (i = 0; i < sc->vmx_nintrs; i++) {
+ pci_intr_disestablish(pc, sc->vmx_ihs[i]);
+ }
+ pci_intr_release(pc, sc->vmx_intrs, sc->vmx_nintrs);
+}
+
+int
+vmxnet3_setup_msix_interrupts(struct vmxnet3_softc *sc)
+{
+ pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_rxqueue *rxq;
+ pci_intr_handle_t *intr;
+ void **ihs;
+ int intr_idx, i;
+ const char *intrstr;
+ char intrbuf[PCI_INTRSTR_LEN];
+ char xnamebuf[32];
+
+ intr = sc->vmx_intrs;
+ intr_idx = 0;
+ ihs = sc->vmx_ihs;
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++, intr++, ihs++, intr_idx++) {
+ snprintf(xnamebuf, 32, "%s: tx %d", device_xname(sc->vmx_dev), i);
+
+ txq = &sc->vmx_txq[i];
+
+ intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
+
+ pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
+ *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
+ vmxnet3_txq_intr, txq, xnamebuf);
+ if (*ihs == NULL) {
+ aprint_error_dev(sc->vmx_dev,
+ "unable to establish tx interrupt at %s\n", intrstr);
+ return (-1);
+ }
+ aprint_normal_dev(sc->vmx_dev, "tx interrupting at %s\n", intrstr);
+
+ txq->vxtxq_intr_idx = intr_idx;
+ }
+
+ for (i = 0; i < sc->vmx_nrxqueues; i++, intr++, ihs++, intr_idx++) {
+ snprintf(xnamebuf, 32, "%s: rx %d", device_xname(sc->vmx_dev), i);
+
+ rxq = &sc->vmx_rxq[i];
+
+ intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
+
+ pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
+ *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
+ vmxnet3_rxq_intr, rxq, xnamebuf);
+ if (*ihs == NULL) {
+ aprint_error_dev(sc->vmx_dev,
+ "unable to establish rx interrupt at %s\n", intrstr);
+ return (-1);
+ }
+ aprint_normal_dev(sc->vmx_dev, "rx interrupting at %s\n", intrstr);
+
+ rxq->vxrxq_intr_idx = intr_idx;
+ }
+
+ intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
+
+ snprintf(xnamebuf, 32, "%s: link", device_xname(sc->vmx_dev));
+ pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
+ *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
+ vmxnet3_event_intr, sc, xnamebuf);
+ if (*ihs == NULL) {
+ aprint_error_dev(sc->vmx_dev,
+ "unable to establish event interrupt at %s\n", intrstr);
+ return (-1);
+ }
+ aprint_normal_dev(sc->vmx_dev, "event interrupting at %s\n", intrstr);
+
+ sc->vmx_event_intr_idx = intr_idx;
+
+ return (0);
+}
+
+int
+vmxnet3_setup_msi_interrupt(struct vmxnet3_softc *sc)
+{
+ pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
+ pci_intr_handle_t *intr;
+ void **ihs;
+ int i;
+ const char *intrstr;
+ char intrbuf[PCI_INTRSTR_LEN];
+ char xnamebuf[32];
+
+ intr = &sc->vmx_intrs[0];
+ ihs = sc->vmx_ihs;
+
+ intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
+
+ snprintf(xnamebuf, 32, "%s: msi", device_xname(sc->vmx_dev));
+ pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
+ *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
+ vmxnet3_legacy_intr, sc, xnamebuf);
+ if (*ihs == NULL) {
+ aprint_error_dev(sc->vmx_dev,
+ "unable to establish interrupt at %s\n", intrstr);
+ return (-1);
+ }
+ aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++)
+ sc->vmx_txq[i].vxtxq_intr_idx = 0;
+ for (i = 0; i < sc->vmx_nrxqueues; i++)
+ sc->vmx_rxq[i].vxrxq_intr_idx = 0;
+ sc->vmx_event_intr_idx = 0;
+
+ return (0);
+}
+
+int
+vmxnet3_setup_legacy_interrupt(struct vmxnet3_softc *sc)
+{
+ pci_chipset_tag_t pc = sc->vmx_pa->pa_pc;
+ pci_intr_handle_t *intr;
+ void **ihs;
+ int i;
+ const char *intrstr;
+ char intrbuf[PCI_INTRSTR_LEN];
+ char xnamebuf[32];
+
+ intr = &sc->vmx_intrs[0];
+ ihs = sc->vmx_ihs;
+
+ intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
+
+ snprintf(xnamebuf, 32, "%s:legacy", device_xname(sc->vmx_dev));
+ pci_intr_setattr(pc, intr, PCI_INTR_MPSAFE, true);
+ *ihs = pci_intr_establish_xname(pc, *intr, IPL_NET,
+ vmxnet3_legacy_intr, sc, xnamebuf);
+ if (*ihs == NULL) {
+ aprint_error_dev(sc->vmx_dev,
+ "unable to establish interrupt at %s\n", intrstr);
+ return (-1);
+ }
+ aprint_normal_dev(sc->vmx_dev, "interrupting at %s\n", intrstr);
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++)
+ sc->vmx_txq[i].vxtxq_intr_idx = 0;
+ for (i = 0; i < sc->vmx_nrxqueues; i++)
+ sc->vmx_rxq[i].vxrxq_intr_idx = 0;
+ sc->vmx_event_intr_idx = 0;
+
+ return (0);
+}
+
+void
+vmxnet3_set_interrupt_idx(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txq_shared *txs;
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_rxq_shared *rxs;
+ int i;
+
+ sc->vmx_ds->evintr = sc->vmx_event_intr_idx;
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++) {
+ txq = &sc->vmx_txq[i];
+ txs = txq->vxtxq_ts;
+ txs->intr_idx = txq->vxtxq_intr_idx;
+ }
+
+ for (i = 0; i < sc->vmx_nrxqueues; i++) {
+ rxq = &sc->vmx_rxq[i];
+ rxs = rxq->vxrxq_rs;
+ rxs->intr_idx = rxq->vxrxq_intr_idx;
+ }
+}
+
+int
+vmxnet3_setup_interrupts(struct vmxnet3_softc *sc)
+{
+ int error;
+
+ switch (sc->vmx_intr_type) {
+ case VMXNET3_IT_MSIX:
+ error = vmxnet3_setup_msix_interrupts(sc);
+ break;
+ case VMXNET3_IT_MSI:
+ error = vmxnet3_setup_msi_interrupt(sc);
+ break;
+ case VMXNET3_IT_LEGACY:
+ error = vmxnet3_setup_legacy_interrupt(sc);
+ break;
+ default:
+ panic("%s: invalid interrupt type %d", __func__,
+ sc->vmx_intr_type);
+ }
+
+ if (error == 0)
+ vmxnet3_set_interrupt_idx(sc);
+
+ return (error);
+}
+
+int
+vmxnet3_init_rxq(struct vmxnet3_softc *sc, int q)
+{
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_rxring *rxr;
+ int i;
+
+ rxq = &sc->vmx_rxq[q];
+
+ snprintf(rxq->vxrxq_name, sizeof(rxq->vxrxq_name), "%s-rx%d",
+ device_xname(sc->vmx_dev), q);
+ rxq->vxrxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);
+
+ rxq->vxrxq_sc = sc;
+ rxq->vxrxq_id = q;
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+ rxr->vxrxr_rid = i;
+ rxr->vxrxr_ndesc = sc->vmx_nrxdescs;
+ rxr->vxrxr_rxbuf = kmem_zalloc(rxr->vxrxr_ndesc *
+ sizeof(struct vmxnet3_rxbuf), KM_SLEEP);
+ if (rxr->vxrxr_rxbuf == NULL)
+ return (ENOMEM);
+
+ rxq->vxrxq_comp_ring.vxcr_ndesc += sc->vmx_nrxdescs;
+ }
+
+ return (0);
+}
+
+int
+vmxnet3_init_txq(struct vmxnet3_softc *sc, int q)
+{
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txring *txr;
+
+ txq = &sc->vmx_txq[q];
+ txr = &txq->vxtxq_cmd_ring;
+
+ snprintf(txq->vxtxq_name, sizeof(txq->vxtxq_name), "%s-tx%d",
+ device_xname(sc->vmx_dev), q);
+ txq->vxtxq_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET /* XXX */);
+
+ txq->vxtxq_sc = sc;
+ txq->vxtxq_id = q;
+
+ txr->vxtxr_ndesc = sc->vmx_ntxdescs;
+ txr->vxtxr_txbuf = kmem_zalloc(txr->vxtxr_ndesc *
+ sizeof(struct vmxnet3_txbuf), KM_SLEEP);
+ if (txr->vxtxr_txbuf == NULL)
+ return (ENOMEM);
+
+ txq->vxtxq_comp_ring.vxcr_ndesc = sc->vmx_ntxdescs;
+
+ return (0);
+}
+
+int
+vmxnet3_alloc_rxtx_queues(struct vmxnet3_softc *sc)
+{
+ int i, error;
+
+ KASSERT(!cpu_intr_p());
+ KASSERT(!cpu_softintr_p());
+
+ /*
+ * Only attempt to create multiple queues if MSIX is available.
+ * This check prevents us from allocating queue structures that
+ * we will not use.
+ *
+ * FreeBSD:
+ * MSIX is disabled by default because its apparently broken for
+ * devices passed through by at least ESXi 5.1.
+ * The hw.pci.honor_msi_blacklist tunable must be set to zero for MSIX.
+ */
+ if (sc->vmx_flags & VMXNET3_FLAG_NO_MSIX) {
+ sc->vmx_max_nrxqueues = 1;
+ sc->vmx_max_ntxqueues = 1;
+ }
+
+ sc->vmx_rxq = kmem_zalloc(
+ sizeof(struct vmxnet3_rxqueue) * sc->vmx_max_nrxqueues, KM_SLEEP);
+ sc->vmx_txq = kmem_zalloc(
+ sizeof(struct vmxnet3_txqueue) * sc->vmx_max_ntxqueues, KM_SLEEP);
+ if (sc->vmx_rxq == NULL || sc->vmx_txq == NULL)
+ return (ENOMEM);
+
+ for (i = 0; i < sc->vmx_max_nrxqueues; i++) {
+ error = vmxnet3_init_rxq(sc, i);
+ if (error)
+ return (error);
+ }
+
+ for (i = 0; i < sc->vmx_max_ntxqueues; i++) {
+ error = vmxnet3_init_txq(sc, i);
+ if (error)
+ return (error);
+ }
+
+ return (0);
+}
+
+void
+vmxnet3_destroy_rxq(struct vmxnet3_rxqueue *rxq)
+{
+ struct vmxnet3_rxring *rxr;
+ int i;
+
+ rxq->vxrxq_sc = NULL;
+ rxq->vxrxq_id = -1;
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ if (rxr->vxrxr_rxbuf != NULL) {
+ kmem_free(rxr->vxrxr_rxbuf,
+ rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxbuf));
+ rxr->vxrxr_rxbuf = NULL;
+ }
+ }
+
+ if (rxq->vxrxq_mtx != NULL)
+ mutex_obj_free(rxq->vxrxq_mtx);
+}
+
+void
+vmxnet3_destroy_txq(struct vmxnet3_txqueue *txq)
+{
+ struct vmxnet3_txring *txr;
+
+ txr = &txq->vxtxq_cmd_ring;
+
+ txq->vxtxq_sc = NULL;
+ txq->vxtxq_id = -1;
+
+ if (txr->vxtxr_txbuf != NULL) {
+ kmem_free(txr->vxtxr_txbuf,
+ txr->vxtxr_ndesc * sizeof(struct vmxnet3_txbuf));
+ txr->vxtxr_txbuf = NULL;
+ }
+
+ if (txq->vxtxq_mtx != NULL)
+ mutex_obj_free(txq->vxtxq_mtx);
+}
+
+void
+vmxnet3_free_rxtx_queues(struct vmxnet3_softc *sc)
+{
+ int i;
+
+ if (sc->vmx_rxq != NULL) {
+ for (i = 0; i < sc->vmx_max_nrxqueues; i++)
+ vmxnet3_destroy_rxq(&sc->vmx_rxq[i]);
+ kmem_free(sc->vmx_rxq,
+ sizeof(struct vmxnet3_rxqueue) * sc->vmx_max_nrxqueues);
+ sc->vmx_rxq = NULL;
+ }
+
+ if (sc->vmx_txq != NULL) {
+ for (i = 0; i < sc->vmx_max_ntxqueues; i++)
+ vmxnet3_destroy_txq(&sc->vmx_txq[i]);
+ kmem_free(sc->vmx_txq,
+ sizeof(struct vmxnet3_txqueue) * sc->vmx_max_ntxqueues);
+ sc->vmx_txq = NULL;
+ }
+}
+
+int
+vmxnet3_alloc_shared_data(struct vmxnet3_softc *sc)
+{
+ device_t dev;
+ uint8_t *kva;
+ size_t size;
+ int i, error;
+
+ dev = sc->vmx_dev;
+
+ size = sizeof(struct vmxnet3_driver_shared);
+ error = vmxnet3_dma_malloc(sc, size, 1, &sc->vmx_ds_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc shared memory\n");
+ return (error);
+ }
+ sc->vmx_ds = (struct vmxnet3_driver_shared *) sc->vmx_ds_dma.dma_vaddr;
+
+ size = sc->vmx_ntxqueues * sizeof(struct vmxnet3_txq_shared) +
+ sc->vmx_nrxqueues * sizeof(struct vmxnet3_rxq_shared);
+ error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_qs_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc queue shared memory\n");
+ return (error);
+ }
+ sc->vmx_qs = (void *) sc->vmx_qs_dma.dma_vaddr;
+ kva = sc->vmx_qs;
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++) {
+ sc->vmx_txq[i].vxtxq_ts = (struct vmxnet3_txq_shared *) kva;
+ kva += sizeof(struct vmxnet3_txq_shared);
+ }
+ for (i = 0; i < sc->vmx_nrxqueues; i++) {
+ sc->vmx_rxq[i].vxrxq_rs = (struct vmxnet3_rxq_shared *) kva;
+ kva += sizeof(struct vmxnet3_rxq_shared);
+ }
+
+ if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+ size = sizeof(struct vmxnet3_rss_shared);
+ error = vmxnet3_dma_malloc(sc, size, 128, &sc->vmx_rss_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc rss shared memory\n");
+ return (error);
+ }
+ sc->vmx_rss =
+ (struct vmxnet3_rss_shared *) sc->vmx_rss_dma.dma_vaddr;
+ }
+
+ return (0);
+}
+
+void
+vmxnet3_free_shared_data(struct vmxnet3_softc *sc)
+{
+
+ if (sc->vmx_rss != NULL) {
+ vmxnet3_dma_free(sc, &sc->vmx_rss_dma);
+ sc->vmx_rss = NULL;
+ }
+
+ if (sc->vmx_qs != NULL) {
+ vmxnet3_dma_free(sc, &sc->vmx_qs_dma);
+ sc->vmx_qs = NULL;
+ }
+
+ if (sc->vmx_ds != NULL) {
+ vmxnet3_dma_free(sc, &sc->vmx_ds_dma);
+ sc->vmx_ds = NULL;
+ }
+}
+
+int
+vmxnet3_alloc_txq_data(struct vmxnet3_softc *sc)
+{
+ device_t dev;
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_comp_ring *txc;
+ size_t descsz, compsz;
+ int i, q, error;
+
+ dev = sc->vmx_dev;
+
+ for (q = 0; q < sc->vmx_ntxqueues; q++) {
+ txq = &sc->vmx_txq[q];
+ txr = &txq->vxtxq_cmd_ring;
+ txc = &txq->vxtxq_comp_ring;
+
+ descsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc);
+ compsz = txr->vxtxr_ndesc * sizeof(struct vmxnet3_txcompdesc);
+
+ error = vmxnet3_dma_malloc(sc, descsz, 512, &txr->vxtxr_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc Tx descriptors for "
+ "queue %d error %d\n", q, error);
+ return (error);
+ }
+ txr->vxtxr_txd =
+ (struct vmxnet3_txdesc *) txr->vxtxr_dma.dma_vaddr;
+
+ error = vmxnet3_dma_malloc(sc, compsz, 512, &txc->vxcr_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc Tx comp descriptors "
+ "for queue %d error %d\n", q, error);
+ return (error);
+ }
+ txc->vxcr_u.txcd =
+ (struct vmxnet3_txcompdesc *) txc->vxcr_dma.dma_vaddr;
+
+ for (i = 0; i < txr->vxtxr_ndesc; i++) {
+ error = bus_dmamap_create(sc->vmx_dmat, VMXNET3_TX_MAXSIZE,
+ VMXNET3_TX_MAXSEGS, VMXNET3_TX_MAXSEGSIZE, 0, BUS_DMA_NOWAIT,
+ &txr->vxtxr_txbuf[i].vtxb_dmamap);
+ if (error) {
+ device_printf(dev, "unable to create Tx buf "
+ "dmamap for queue %d idx %d\n", q, i);
+ return (error);
+ }
+ }
+ }
+
+ return (0);
+}
+
+void
+vmxnet3_free_txq_data(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_comp_ring *txc;
+ struct vmxnet3_txbuf *txb;
+ int i, q;
+
+ for (q = 0; q < sc->vmx_ntxqueues; q++) {
+ txq = &sc->vmx_txq[q];
+ txr = &txq->vxtxq_cmd_ring;
+ txc = &txq->vxtxq_comp_ring;
+
+ for (i = 0; i < txr->vxtxr_ndesc; i++) {
+ txb = &txr->vxtxr_txbuf[i];
+ if (txb->vtxb_dmamap != NULL) {
+ bus_dmamap_destroy(sc->vmx_dmat,
+ txb->vtxb_dmamap);
+ txb->vtxb_dmamap = NULL;
+ }
+ }
+
+ if (txc->vxcr_u.txcd != NULL) {
+ vmxnet3_dma_free(sc, &txc->vxcr_dma);
+ txc->vxcr_u.txcd = NULL;
+ }
+
+ if (txr->vxtxr_txd != NULL) {
+ vmxnet3_dma_free(sc, &txr->vxtxr_dma);
+ txr->vxtxr_txd = NULL;
+ }
+ }
+}
+
+int
+vmxnet3_alloc_rxq_data(struct vmxnet3_softc *sc)
+{
+ device_t dev;
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_comp_ring *rxc;
+ int descsz, compsz;
+ int i, j, q, error;
+
+ dev = sc->vmx_dev;
+
+ for (q = 0; q < sc->vmx_nrxqueues; q++) {
+ rxq = &sc->vmx_rxq[q];
+ rxc = &rxq->vxrxq_comp_ring;
+ compsz = 0;
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ descsz = rxr->vxrxr_ndesc *
+ sizeof(struct vmxnet3_rxdesc);
+ compsz += rxr->vxrxr_ndesc *
+ sizeof(struct vmxnet3_rxcompdesc);
+
+ error = vmxnet3_dma_malloc(sc, descsz, 512,
+ &rxr->vxrxr_dma);
+ if (error) {
+ device_printf(dev, "cannot allocate Rx "
+ "descriptors for queue %d/%d error %d\n",
+ i, q, error);
+ return (error);
+ }
+ rxr->vxrxr_rxd =
+ (struct vmxnet3_rxdesc *) rxr->vxrxr_dma.dma_vaddr;
+ }
+
+ error = vmxnet3_dma_malloc(sc, compsz, 512, &rxc->vxcr_dma);
+ if (error) {
+ device_printf(dev, "cannot alloc Rx comp descriptors "
+ "for queue %d error %d\n", q, error);
+ return (error);
+ }
+ rxc->vxcr_u.rxcd =
+ (struct vmxnet3_rxcompdesc *) rxc->vxcr_dma.dma_vaddr;
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
+ JUMBO_LEN, 0, BUS_DMA_NOWAIT,
+ &rxr->vxrxr_spare_dmap);
+ if (error) {
+ device_printf(dev, "unable to create spare "
+ "dmamap for queue %d/%d error %d\n",
+ q, i, error);
+ return (error);
+ }
+
+ for (j = 0; j < rxr->vxrxr_ndesc; j++) {
+ error = bus_dmamap_create(sc->vmx_dmat, JUMBO_LEN, 1,
+ JUMBO_LEN, 0, BUS_DMA_NOWAIT,
+ &rxr->vxrxr_rxbuf[j].vrxb_dmamap);
+ if (error) {
+ device_printf(dev, "unable to create "
+ "dmamap for queue %d/%d slot %d "
+ "error %d\n",
+ q, i, j, error);
+ return (error);
+ }
+ }
+ }
+ }
+
+ return (0);
+}
+
+void
+vmxnet3_free_rxq_data(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_comp_ring *rxc;
+ struct vmxnet3_rxbuf *rxb;
+ int i, j, q;
+
+ for (q = 0; q < sc->vmx_nrxqueues; q++) {
+ rxq = &sc->vmx_rxq[q];
+ rxc = &rxq->vxrxq_comp_ring;
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ if (rxr->vxrxr_spare_dmap != NULL) {
+ bus_dmamap_destroy(sc->vmx_dmat,
+ rxr->vxrxr_spare_dmap);
+ rxr->vxrxr_spare_dmap = NULL;
+ }
+
+ for (j = 0; j < rxr->vxrxr_ndesc; j++) {
+ rxb = &rxr->vxrxr_rxbuf[j];
+ if (rxb->vrxb_dmamap != NULL) {
+ bus_dmamap_destroy(sc->vmx_dmat,
+ rxb->vrxb_dmamap);
+ rxb->vrxb_dmamap = NULL;
+ }
+ }
+ }
+
+ if (rxc->vxcr_u.rxcd != NULL) {
+ vmxnet3_dma_free(sc, &rxc->vxcr_dma);
+ rxc->vxcr_u.rxcd = NULL;
+ }
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ if (rxr->vxrxr_rxd != NULL) {
+ vmxnet3_dma_free(sc, &rxr->vxrxr_dma);
+ rxr->vxrxr_rxd = NULL;
+ }
+ }
+ }
+}
+
+int
+vmxnet3_alloc_queue_data(struct vmxnet3_softc *sc)
+{
+ int error;
+
+ error = vmxnet3_alloc_txq_data(sc);
+ if (error)
+ return (error);
+
+ error = vmxnet3_alloc_rxq_data(sc);
+ if (error)
+ return (error);
+
+ return (0);
+}
+
+void
+vmxnet3_free_queue_data(struct vmxnet3_softc *sc)
+{
+
+ if (sc->vmx_rxq != NULL)
+ vmxnet3_free_rxq_data(sc);
+
+ if (sc->vmx_txq != NULL)
+ vmxnet3_free_txq_data(sc);
+}
+
+int
+vmxnet3_alloc_mcast_table(struct vmxnet3_softc *sc)
+{
+ int error;
+
+ error = vmxnet3_dma_malloc(sc, VMXNET3_MULTICAST_MAX * ETHER_ADDR_LEN,
+ 32, &sc->vmx_mcast_dma);
+ if (error)
+ device_printf(sc->vmx_dev, "unable to alloc multicast table\n");
+ else
+ sc->vmx_mcast = sc->vmx_mcast_dma.dma_vaddr;
+
+ return (error);
+}
+
+void
+vmxnet3_free_mcast_table(struct vmxnet3_softc *sc)
+{
+
+ if (sc->vmx_mcast != NULL) {
+ vmxnet3_dma_free(sc, &sc->vmx_mcast_dma);
+ sc->vmx_mcast = NULL;
+ }
+}
+
+void
+vmxnet3_init_shared_data(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_driver_shared *ds;
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txq_shared *txs;
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_rxq_shared *rxs;
+ int i;
+
+ ds = sc->vmx_ds;
+
+ /*
+ * Initialize fields of the shared data that remains the same across
+ * reinits. Note the shared data is zero'd when allocated.
+ */
+
+ ds->magic = VMXNET3_REV1_MAGIC;
+
+ /* DriverInfo */
+ ds->version = VMXNET3_DRIVER_VERSION;
+ ds->guest = VMXNET3_GOS_FREEBSD |
+#ifdef __LP64__
+ VMXNET3_GOS_64BIT;
+#else
+ VMXNET3_GOS_32BIT;
+#endif
+ ds->vmxnet3_revision = 1;
+ ds->upt_version = 1;
+
+ /* Misc. conf */
+ ds->driver_data = vtophys(sc);
+ ds->driver_data_len = sizeof(struct vmxnet3_softc);
+ ds->queue_shared = sc->vmx_qs_dma.dma_paddr;
+ ds->queue_shared_len = sc->vmx_qs_dma.dma_size;
+ ds->nrxsg_max = sc->vmx_max_rxsegs;
+
+ /* RSS conf */
+ if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+ ds->rss.version = 1;
+ ds->rss.paddr = sc->vmx_rss_dma.dma_paddr;
+ ds->rss.len = sc->vmx_rss_dma.dma_size;
+ }
+
+ /* Interrupt control. */
+ ds->automask = sc->vmx_intr_mask_mode == VMXNET3_IMM_AUTO;
+ ds->nintr = sc->vmx_nintrs;
+ ds->evintr = sc->vmx_event_intr_idx;
+ ds->ictrl = VMXNET3_ICTRL_DISABLE_ALL;
+
+ for (i = 0; i < sc->vmx_nintrs; i++)
+ ds->modlevel[i] = UPT1_IMOD_ADAPTIVE;
+
+ /* Receive filter. */
+ ds->mcast_table = sc->vmx_mcast_dma.dma_paddr;
+ ds->mcast_tablelen = sc->vmx_mcast_dma.dma_size;
+
+ /* Tx queues */
+ for (i = 0; i < sc->vmx_ntxqueues; i++) {
+ txq = &sc->vmx_txq[i];
+ txs = txq->vxtxq_ts;
+
+ txs->cmd_ring = txq->vxtxq_cmd_ring.vxtxr_dma.dma_paddr;
+ txs->cmd_ring_len = txq->vxtxq_cmd_ring.vxtxr_ndesc;
+ txs->comp_ring = txq->vxtxq_comp_ring.vxcr_dma.dma_paddr;
+ txs->comp_ring_len = txq->vxtxq_comp_ring.vxcr_ndesc;
+ txs->driver_data = vtophys(txq);
+ txs->driver_data_len = sizeof(struct vmxnet3_txqueue);
+ }
+
+ /* Rx queues */
+ for (i = 0; i < sc->vmx_nrxqueues; i++) {
+ rxq = &sc->vmx_rxq[i];
+ rxs = rxq->vxrxq_rs;
+
+ rxs->cmd_ring[0] = rxq->vxrxq_cmd_ring[0].vxrxr_dma.dma_paddr;
+ rxs->cmd_ring_len[0] = rxq->vxrxq_cmd_ring[0].vxrxr_ndesc;
+ rxs->cmd_ring[1] = rxq->vxrxq_cmd_ring[1].vxrxr_dma.dma_paddr;
+ rxs->cmd_ring_len[1] = rxq->vxrxq_cmd_ring[1].vxrxr_ndesc;
+ rxs->comp_ring = rxq->vxrxq_comp_ring.vxcr_dma.dma_paddr;
+ rxs->comp_ring_len = rxq->vxrxq_comp_ring.vxcr_ndesc;
+ rxs->driver_data = vtophys(rxq);
+ rxs->driver_data_len = sizeof(struct vmxnet3_rxqueue);
+ }
+}
+
+void
+vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *sc)
+{
+ /*
+ * Use the same key as the Linux driver until FreeBSD can do
+ * RSS (presumably Toeplitz) in software.
+ */
+ static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
+ 0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
+ 0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
+ 0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
+ 0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
+ 0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
+ };
+
+ struct vmxnet3_rss_shared *rss;
+ int i;
+
+ rss = sc->vmx_rss;
+
+ rss->hash_type =
+ UPT1_RSS_HASH_TYPE_IPV4 | UPT1_RSS_HASH_TYPE_TCP_IPV4 |
+ UPT1_RSS_HASH_TYPE_IPV6 | UPT1_RSS_HASH_TYPE_TCP_IPV6;
+ rss->hash_func = UPT1_RSS_HASH_FUNC_TOEPLITZ;
+ rss->hash_key_size = UPT1_RSS_MAX_KEY_SIZE;
+ rss->ind_table_size = UPT1_RSS_MAX_IND_TABLE_SIZE;
+ memcpy(rss->hash_key, rss_key, UPT1_RSS_MAX_KEY_SIZE);
+
+ for (i = 0; i < UPT1_RSS_MAX_IND_TABLE_SIZE; i++)
+ rss->ind_table[i] = i % sc->vmx_nrxqueues;
+}
+
+void
+vmxnet3_reinit_shared_data(struct vmxnet3_softc *sc)
+{
+ struct ifnet *ifp;
+ struct vmxnet3_driver_shared *ds;
+
+ ifp = &sc->vmx_ethercom.ec_if;
+ ds = sc->vmx_ds;
+
+ ds->mtu = ifp->if_mtu;
+ ds->ntxqueue = sc->vmx_ntxqueues;
+ ds->nrxqueue = sc->vmx_nrxqueues;
+
+ ds->upt_features = 0;
+ if (ifp->if_capenable &
+ (IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
+ IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx))
+ ds->upt_features |= UPT1_F_CSUM;
+ if (sc->vmx_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING)
+ ds->upt_features |= UPT1_F_VLAN;
+
+ if (sc->vmx_flags & VMXNET3_FLAG_RSS) {
+ ds->upt_features |= UPT1_F_RSS;
+ vmxnet3_reinit_rss_shared_data(sc);
+ }
+
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSL, sc->vmx_ds_dma.dma_paddr);
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_DSH,
+ (uint64_t) sc->vmx_ds_dma.dma_paddr >> 32);
+}
+
+int
+vmxnet3_alloc_data(struct vmxnet3_softc *sc)
+{
+ int error;
+
+ error = vmxnet3_alloc_shared_data(sc);
+ if (error)
+ return (error);
+
+ error = vmxnet3_alloc_queue_data(sc);
+ if (error)
+ return (error);
+
+ error = vmxnet3_alloc_mcast_table(sc);
+ if (error)
+ return (error);
+
+ vmxnet3_init_shared_data(sc);
+
+ return (0);
+}
+
+void
+vmxnet3_free_data(struct vmxnet3_softc *sc)
+{
+
+ vmxnet3_free_mcast_table(sc);
+ vmxnet3_free_queue_data(sc);
+ vmxnet3_free_shared_data(sc);
+}
+
+int
+vmxnet3_setup_interface(struct vmxnet3_softc *sc)
+{
+ struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
+
+ vmxnet3_get_lladdr(sc);
+ aprint_normal_dev(sc->vmx_dev, "Ethernet address %s\n",
+ ether_sprintf(sc->vmx_lladdr));
+ vmxnet3_set_lladdr(sc);
+
+ strlcpy(ifp->if_xname, device_xname(sc->vmx_dev), IFNAMSIZ);
+ ifp->if_softc = sc;
+ ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
+ ifp->if_ioctl = vmxnet3_ioctl;
+ ifp->if_start = vmxnet3_start;
+ ifp->if_watchdog = NULL;
+ ifp->if_init = vmxnet3_init;
+ ifp->if_stop = vmxnet3_stop;
+ sc->vmx_ethercom.ec_if.if_capabilities |=IFCAP_CSUM_IPv4_Rx |
+ IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
+ IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
+ IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
+ IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
+
+ ifp->if_capenable = ifp->if_capabilities;
+
+ sc->vmx_ethercom.ec_if.if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
+
+ sc->vmx_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING;
+
+ IFQ_SET_MAXLEN(&ifp->if_snd, sc->vmx_ntxdescs);
+ IFQ_SET_READY(&ifp->if_snd);
+
+ ifmedia_init(&sc->vmx_media, IFM_IMASK, vmxnet3_media_change,
+ vmxnet3_media_status);
+ ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_AUTO, 0, NULL);
+ ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_10G_T|IFM_FDX, 0, NULL);
+ ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_10G_T, 0, NULL);
+ ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_1000_T|IFM_FDX, 0, NULL);
+ ifmedia_add(&sc->vmx_media, IFM_ETHER|IFM_1000_T, 0, NULL);
+ ifmedia_set(&sc->vmx_media, IFM_ETHER|IFM_AUTO);
+
+ if_attach(ifp);
+ ether_ifattach(ifp, sc->vmx_lladdr);
+ ether_set_ifflags_cb(&sc->vmx_ethercom, vmxnet3_ifflags_cb);
+ vmxnet3_link_status(sc);
+
+ return (0);
+}
+
+void
+vmxnet3_evintr(struct vmxnet3_softc *sc)
+{
+ device_t dev;
+ struct ifnet *ifp;
+ struct vmxnet3_txq_shared *ts;
+ struct vmxnet3_rxq_shared *rs;
+ uint32_t event;
+ int reset;
+
+ dev = sc->vmx_dev;
+ ifp = &sc->vmx_ethercom.ec_if;
+ reset = 0;
+
+ VMXNET3_CORE_LOCK(sc);
+
+ /* Clear events. */
+ event = sc->vmx_ds->event;
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_EVENT, event);
+
+ if (event & VMXNET3_EVENT_LINK) {
+ vmxnet3_link_status(sc);
+ if (sc->vmx_link_active != 0)
+ vmxnet3_start(&sc->vmx_ethercom.ec_if);
+ }
+
+ if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
+ reset = 1;
+ vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_STATUS);
+ ts = sc->vmx_txq[0].vxtxq_ts;
+ if (ts->stopped != 0)
+ device_printf(dev, "Tx queue error %#x\n", ts->error);
+ rs = sc->vmx_rxq[0].vxrxq_rs;
+ if (rs->stopped != 0)
+ device_printf(dev, "Rx queue error %#x\n", rs->error);
+ device_printf(dev, "Rx/Tx queue error event ... resetting\n");
+ }
+
+ if (event & VMXNET3_EVENT_DIC)
+ device_printf(dev, "device implementation change event\n");
+ if (event & VMXNET3_EVENT_DEBUG)
+ device_printf(dev, "debug event\n");
+
+ if (reset != 0) {
+ ifp->if_flags &= ~IFF_RUNNING;
+ vmxnet3_init_locked(sc);
+ }
+
+ VMXNET3_CORE_UNLOCK(sc);
+}
+
+void
+vmxnet3_txq_eof(struct vmxnet3_txqueue *txq)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_comp_ring *txc;
+ struct vmxnet3_txcompdesc *txcd;
+ struct vmxnet3_txbuf *txb;
+ struct mbuf *m;
+ u_int sop;
+
+ sc = txq->vxtxq_sc;
+ txr = &txq->vxtxq_cmd_ring;
+ txc = &txq->vxtxq_comp_ring;
+
+ VMXNET3_TXQ_LOCK_ASSERT(txq);
+
+ for (;;) {
+ txcd = &txc->vxcr_u.txcd[txc->vxcr_next];
+ if (txcd->gen != txc->vxcr_gen)
+ break;
+ vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
+
+ if (++txc->vxcr_next == txc->vxcr_ndesc) {
+ txc->vxcr_next = 0;
+ txc->vxcr_gen ^= 1;
+ }
+
+ sop = txr->vxtxr_next;
+ txb = &txr->vxtxr_txbuf[sop];
+
+ if ((m = txb->vtxb_m) != NULL) {
+ bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
+ 0, txb->vtxb_dmamap->dm_mapsize,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);
+
+ txq->vxtxq_stats.vmtxs_opackets++;
+ txq->vxtxq_stats.vmtxs_obytes += m->m_pkthdr.len;
+ if (m->m_flags & M_MCAST)
+ txq->vxtxq_stats.vmtxs_omcasts++;
+
+ m_freem(m);
+ txb->vtxb_m = NULL;
+ }
+
+ txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
+ }
+
+ if (txr->vxtxr_head == txr->vxtxr_next)
+ txq->vxtxq_watchdog = 0;
+}
+
+int
+vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
+{
+ struct mbuf *m;
+ struct vmxnet3_rxdesc *rxd;
+ struct vmxnet3_rxbuf *rxb;
+ bus_dma_tag_t tag;
+ bus_dmamap_t dmap;
+ int idx, btype, error;
+
+ tag = sc->vmx_dmat;
+ dmap = rxr->vxrxr_spare_dmap;
+ idx = rxr->vxrxr_fill;
+ rxd = &rxr->vxrxr_rxd[idx];
+ rxb = &rxr->vxrxr_rxbuf[idx];
+
+ /* Don't allocate buffers for ring 2 for now. */
+ if (rxr->vxrxr_rid != 0)
+ return -1;
+ btype = VMXNET3_BTYPE_HEAD;
+
+ MGETHDR(m, M_DONTWAIT, MT_DATA);
+ if (m == NULL)
+ return (ENOBUFS);
+
+ MCLGET(m, M_DONTWAIT);
+ if ((m->m_flags & M_EXT) == 0) {
+ sc->vmx_stats.vmst_mgetcl_failed++;
+ m_freem(m);
+ return (ENOBUFS);
+ }
+
+ m->m_pkthdr.len = m->m_len = JUMBO_LEN;
+ m_adj(m, ETHER_ALIGN);
+
+ error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT);
+ if (error) {
+ m_freem(m);
+ sc->vmx_stats.vmst_mbuf_load_failed++;
+ return (error);
+ }
+
+ if (rxb->vrxb_m != NULL) {
+ bus_dmamap_sync(tag, rxb->vrxb_dmamap,
+ 0, rxb->vrxb_dmamap->dm_mapsize,
+ BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(tag, rxb->vrxb_dmamap);
+ }
+
+ rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
+ rxb->vrxb_dmamap = dmap;
+ rxb->vrxb_m = m;
+
+ rxd->addr = DMAADDR(dmap);
+ rxd->len = m->m_pkthdr.len;
+ rxd->btype = btype;
+ rxd->gen = rxr->vxrxr_gen;
+
+ vmxnet3_rxr_increment_fill(rxr);
+ return (0);
+}
+
+
+
+void
+vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *rxq,
+ struct vmxnet3_rxring *rxr, int idx)
+{
+ struct vmxnet3_rxdesc *rxd;
+
+ rxd = &rxr->vxrxr_rxd[idx];
+ rxd->gen = rxr->vxrxr_gen;
+ vmxnet3_rxr_increment_fill(rxr);
+}
+
+void
+vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *rxq)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_comp_ring *rxc;
+ struct vmxnet3_rxcompdesc *rxcd;
+ int idx, eof;
+
+ sc = rxq->vxrxq_sc;
+ rxc = &rxq->vxrxq_comp_ring;
+
+ do {
+ rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
+ if (rxcd->gen != rxc->vxcr_gen)
+ break; /* Not expected. */
+ vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
+
+ if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
+ rxc->vxcr_next = 0;
+ rxc->vxcr_gen ^= 1;
+ }
+
+ idx = rxcd->rxd_idx;
+ eof = rxcd->eop;
+ if (rxcd->qid < sc->vmx_nrxqueues)
+ rxr = &rxq->vxrxq_cmd_ring[0];
+ else
+ rxr = &rxq->vxrxq_cmd_ring[1];
+ vmxnet3_rxq_eof_discard(rxq, rxr, idx);
+ } while (!eof);
+}
+
+void
+vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
+{
+ if (rxcd->no_csum)
+ return;
+
+ if (rxcd->ipv4) {
+ m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
+ if (rxcd->ipcsum_ok == 0)
+ m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD;
+ }
+
+ if (rxcd->fragment)
+ return;
+
+ if (rxcd->tcp) {
+ m->m_pkthdr.csum_flags |=
+ rxcd->ipv4 ? M_CSUM_TCPv4 : M_CSUM_TCPv6;
+ if ((rxcd->csum_ok) == 0)
+ m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
+ }
+
+ if (rxcd->udp) {
+ m->m_pkthdr.csum_flags |=
+ rxcd->ipv4 ? M_CSUM_UDPv4 : M_CSUM_UDPv6 ;
+ if ((rxcd->csum_ok) == 0)
+ m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
+ }
+}
+
+void
+vmxnet3_rxq_input(struct vmxnet3_rxqueue *rxq,
+ struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
+{
+ struct vmxnet3_softc *sc;
+ struct ifnet *ifp;
+
+ sc = rxq->vxrxq_sc;
+ ifp = &sc->vmx_ethercom.ec_if;
+
+ if (rxcd->error) {
+ rxq->vxrxq_stats.vmrxs_ierrors++;
+ m_freem(m);
+ return;
+ }
+
+ if (!rxcd->no_csum)
+ vmxnet3_rx_csum(rxcd, m);
+ if (rxcd->vlan) {
+ VLAN_INPUT_TAG(ifp, m, rxcd->vtag,
+ rxq->vxrxq_stats.vmrxs_ierrors++;
+ m_freem(m);
+ return);
+ }
+
+ rxq->vxrxq_stats.vmrxs_ipackets++;
+ rxq->vxrxq_stats.vmrxs_ibytes += m->m_pkthdr.len;
+
+ if_percpuq_enqueue(ifp->if_percpuq, m);
+}
+
+void
+vmxnet3_rxq_eof(struct vmxnet3_rxqueue *rxq)
+{
+ struct vmxnet3_softc *sc;
+ struct ifnet *ifp;
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_comp_ring *rxc;
+ struct vmxnet3_rxdesc *rxd;
+ struct vmxnet3_rxcompdesc *rxcd;
+ struct mbuf *m, *m_head, *m_tail;
+ int idx, length;
+
+ sc = rxq->vxrxq_sc;
+ ifp = &sc->vmx_ethercom.ec_if;
+ rxc = &rxq->vxrxq_comp_ring;
+
+ VMXNET3_RXQ_LOCK_ASSERT(rxq);
if ((ifp->if_flags & IFF_RUNNING) == 0)
- return 0;
- if (READ_BAR1(sc, VMXNET3_BAR1_INTR) == 0)
- return 0;
- if (sc->sc_ds->event)
+ return;
+
+ m_head = rxq->vxrxq_mhead;
+ rxq->vxrxq_mhead = NULL;
+ m_tail = rxq->vxrxq_mtail;
+ rxq->vxrxq_mtail = NULL;
+ KASSERT(m_head == NULL || m_tail != NULL);
+
+ for (;;) {
+ rxcd = &rxc->vxcr_u.rxcd[rxc->vxcr_next];
+ if (rxcd->gen != rxc->vxcr_gen) {
+ rxq->vxrxq_mhead = m_head;
+ rxq->vxrxq_mtail = m_tail;
+ break;
+ }
+ vmxnet3_barrier(sc, VMXNET3_BARRIER_RD);
+
+ if (++rxc->vxcr_next == rxc->vxcr_ndesc) {
+ rxc->vxcr_next = 0;
+ rxc->vxcr_gen ^= 1;
+ }
+
+ idx = rxcd->rxd_idx;
+ length = rxcd->len;
+ if (rxcd->qid < sc->vmx_nrxqueues)
+ rxr = &rxq->vxrxq_cmd_ring[0];
+ else
+ rxr = &rxq->vxrxq_cmd_ring[1];
+ rxd = &rxr->vxrxr_rxd[idx];
+
+ m = rxr->vxrxr_rxbuf[idx].vrxb_m;
+ KASSERT(m != NULL);
+
+ /*
+ * The host may skip descriptors. We detect this when this
+ * descriptor does not match the previous fill index. Catch
+ * up with the host now.
+ */
+ if (__predict_false(rxr->vxrxr_fill != idx)) {
+ while (rxr->vxrxr_fill != idx) {
+ rxr->vxrxr_rxd[rxr->vxrxr_fill].gen =
+ rxr->vxrxr_gen;
+ vmxnet3_rxr_increment_fill(rxr);
+ }
+ }
+
+ if (rxcd->sop) {
+ /* start of frame w/o head buffer */
+ KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD);
+ /* start of frame not in ring 0 */
+ KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]);
+ /* duplicate start of frame? */
+ KASSERT(m_head == NULL);
+
+ if (length == 0) {
+ /* Just ignore this descriptor. */
+ vmxnet3_rxq_eof_discard(rxq, rxr, idx);
+ goto nextp;
+ }
+
+ if (vmxnet3_newbuf(sc, rxr) != 0) {
+ rxq->vxrxq_stats.vmrxs_iqdrops++;
+ vmxnet3_rxq_eof_discard(rxq, rxr, idx);
+ if (!rxcd->eop)
+ vmxnet3_rxq_discard_chain(rxq);
+ goto nextp;
+ }
+
+ m_set_rcvif(m, ifp);
+ m->m_pkthdr.len = m->m_len = length;
+ m->m_pkthdr.csum_flags = 0;
+ m_head = m_tail = m;
+
+ } else {
+ /* non start of frame w/o body buffer */
+ KASSERT(rxd->btype == VMXNET3_BTYPE_BODY);
+ /* frame not started? */
+ KASSERT(m_head != NULL);
+
+ if (vmxnet3_newbuf(sc, rxr) != 0) {
+ rxq->vxrxq_stats.vmrxs_iqdrops++;
+ vmxnet3_rxq_eof_discard(rxq, rxr, idx);
+ if (!rxcd->eop)
+ vmxnet3_rxq_discard_chain(rxq);
+ m_freem(m_head);
+ m_head = m_tail = NULL;
+ goto nextp;
+ }
+
+ m->m_len = length;
+ m_head->m_pkthdr.len += length;
+ m_tail->m_next = m;
+ m_tail = m;
+ }
+
+ if (rxcd->eop) {
+ vmxnet3_rxq_input(rxq, rxcd, m_head);
+ m_head = m_tail = NULL;
+
+ /* Must recheck after dropping the Rx lock. */
+ if ((ifp->if_flags & IFF_RUNNING) == 0)
+ break;
+ }
+
+nextp:
+ if (__predict_false(rxq->vxrxq_rs->update_rxhead)) {
+ int qid = rxcd->qid;
+ bus_size_t r;
+
+ idx = (idx + 1) % rxr->vxrxr_ndesc;
+ if (qid >= sc->vmx_nrxqueues) {
+ qid -= sc->vmx_nrxqueues;
+ r = VMXNET3_BAR0_RXH2(qid);
+ } else
+ r = VMXNET3_BAR0_RXH1(qid);
+ vmxnet3_write_bar0(sc, r, idx);
+ }
+ }
+}
+
+int
+vmxnet3_legacy_intr(void *xsc)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_txqueue *txq;
+
+ sc = xsc;
+ rxq = &sc->vmx_rxq[0];
+ txq = &sc->vmx_txq[0];
+
+ if (sc->vmx_intr_type == VMXNET3_IT_LEGACY) {
+ if (vmxnet3_read_bar1(sc, VMXNET3_BAR1_INTR) == 0)
+ return (0);
+ }
+ if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
+ vmxnet3_disable_all_intrs(sc);
+
+ if (sc->vmx_ds->event != 0)
vmxnet3_evintr(sc);
- vmxnet3_rxintr(sc, &sc->sc_rxq[0]);
- vmxnet3_txintr(sc, &sc->sc_txq[0]);
-#ifdef VMXNET3_STAT
- vmxstat.intr++;
-#endif
- vmxnet3_enable_intr(sc, 0);
- return 1;
+
+ VMXNET3_RXQ_LOCK(rxq);
+ vmxnet3_rxq_eof(rxq);
+ VMXNET3_RXQ_UNLOCK(rxq);
+
+ VMXNET3_TXQ_LOCK(txq);
+ vmxnet3_txq_eof(txq);
+ vmxnet3_start_locked(&sc->vmx_ethercom.ec_if);
+ VMXNET3_TXQ_UNLOCK(txq);
+
+ vmxnet3_enable_all_intrs(sc);
+
+ return (1);
+}
+
+int
+vmxnet3_txq_intr(void *xtxq)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_txqueue *txq;
+
+ txq = xtxq;
+ sc = txq->vxtxq_sc;
+
+ if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
+ vmxnet3_disable_intr(sc, txq->vxtxq_intr_idx);
+
+ VMXNET3_TXQ_LOCK(txq);
+ vmxnet3_txq_eof(txq);
+ vmxnet3_start_locked(&sc->vmx_ethercom.ec_if);
+ VMXNET3_TXQ_UNLOCK(txq);
+
+ vmxnet3_enable_intr(sc, txq->vxtxq_intr_idx);
+
+ return (1);
+}
+
+int
+vmxnet3_rxq_intr(void *xrxq)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_rxqueue *rxq;
+
+ rxq = xrxq;
+ sc = rxq->vxrxq_sc;
+
+ if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
+ vmxnet3_disable_intr(sc, rxq->vxrxq_intr_idx);
+
+ VMXNET3_RXQ_LOCK(rxq);
+ vmxnet3_rxq_eof(rxq);
+ VMXNET3_RXQ_UNLOCK(rxq);
+
+ vmxnet3_enable_intr(sc, rxq->vxrxq_intr_idx);
+
+ return (1);
+}
+
+int
+vmxnet3_event_intr(void *xsc)
+{
+ struct vmxnet3_softc *sc;
+
+ sc = xsc;
+
+ if (sc->vmx_intr_mask_mode == VMXNET3_IMM_ACTIVE)
+ vmxnet3_disable_intr(sc, sc->vmx_event_intr_idx);
+
+ if (sc->vmx_ds->event != 0)
+ vmxnet3_evintr(sc);
+
+ vmxnet3_enable_intr(sc, sc->vmx_event_intr_idx);
+
+ return (1);
}
void
-vmxnet3_evintr(struct vmxnet3_softc *sc)
+vmxnet3_txstop(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
{
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- u_int event = sc->sc_ds->event;
- struct vmxnet3_txq_shared *ts;
- struct vmxnet3_rxq_shared *rs;
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_txbuf *txb;
+ int i;
- /* Clear events. */
- WRITE_BAR1(sc, VMXNET3_BAR1_EVENT, event);
+ txr = &txq->vxtxq_cmd_ring;
+
+ for (i = 0; i < txr->vxtxr_ndesc; i++) {
+ txb = &txr->vxtxr_txbuf[i];
+
+ if (txb->vtxb_m == NULL)
+ continue;
+
+ bus_dmamap_sync(sc->vmx_dmat, txb->vtxb_dmamap,
+ 0, txb->vtxb_dmamap->dm_mapsize,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->vmx_dmat, txb->vtxb_dmamap);
+ m_freem(txb->vtxb_m);
+ txb->vtxb_m = NULL;
+ }
+}
+
+void
+vmxnet3_rxstop(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
+{
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_rxbuf *rxb;
+ int i, j;
+
+ if (rxq->vxrxq_mhead != NULL) {
+ m_freem(rxq->vxrxq_mhead);
+ rxq->vxrxq_mhead = NULL;
+ rxq->vxrxq_mtail = NULL;
+ }
+
+ for (i = 0; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+
+ for (j = 0; j < rxr->vxrxr_ndesc; j++) {
+ rxb = &rxr->vxrxr_rxbuf[j];
+
+ if (rxb->vrxb_m == NULL)
+ continue;
+
+ bus_dmamap_sync(sc->vmx_dmat, rxb->vrxb_dmamap,
+ 0, rxb->vrxb_dmamap->dm_mapsize,
+ BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc->vmx_dmat, rxb->vrxb_dmamap);
+ m_freem(rxb->vrxb_m);
+ rxb->vrxb_m = NULL;
+ }
+ }
+}
+
+void
+vmxnet3_stop_rendezvous(struct vmxnet3_softc *sc)
+{
+ struct vmxnet3_rxqueue *rxq;
+ struct vmxnet3_txqueue *txq;
+ int i;
+
+ for (i = 0; i < sc->vmx_nrxqueues; i++) {
+ rxq = &sc->vmx_rxq[i];
+ VMXNET3_RXQ_LOCK(rxq);
+ VMXNET3_RXQ_UNLOCK(rxq);
+ }
+
+ for (i = 0; i < sc->vmx_ntxqueues; i++) {
+ txq = &sc->vmx_txq[i];
+ VMXNET3_TXQ_LOCK(txq);
+ VMXNET3_TXQ_UNLOCK(txq);
+ }
+}
+
+void
+vmxnet3_stop_locked(struct vmxnet3_softc *sc)
+{
+ struct ifnet *ifp;
+ int q;
+
+ ifp = &sc->vmx_ethercom.ec_if;
+ VMXNET3_CORE_LOCK_ASSERT(sc);
+
+ ifp->if_flags &= ~IFF_RUNNING;
+ sc->vmx_link_active = 0;
+ callout_stop(&sc->vmx_tick);
+
+ /* Disable interrupts. */
+ vmxnet3_disable_all_intrs(sc);
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_DISABLE);
+
+ vmxnet3_stop_rendezvous(sc);
+
+ for (q = 0; q < sc->vmx_ntxqueues; q++)
+ vmxnet3_txstop(sc, &sc->vmx_txq[q]);
+ for (q = 0; q < sc->vmx_nrxqueues; q++)
+ vmxnet3_rxstop(sc, &sc->vmx_rxq[q]);
+
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_RESET);
+}
+
+void
+vmxnet3_stop(struct ifnet *ifp, int disable)
+{
+ struct vmxnet3_softc *sc = ifp->if_softc;
+
+ VMXNET3_CORE_LOCK(sc);
+ vmxnet3_stop_locked(sc);
+ VMXNET3_CORE_UNLOCK(sc);
+}
+
+void
+vmxnet3_txinit(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *txq)
+{
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_comp_ring *txc;
+
+ txr = &txq->vxtxq_cmd_ring;
+ txr->vxtxr_head = 0;
+ txr->vxtxr_next = 0;
+ txr->vxtxr_gen = VMXNET3_INIT_GEN;
+ memset(txr->vxtxr_txd, 0,
+ txr->vxtxr_ndesc * sizeof(struct vmxnet3_txdesc));
+
+ txc = &txq->vxtxq_comp_ring;
+ txc->vxcr_next = 0;
+ txc->vxcr_gen = VMXNET3_INIT_GEN;
+ memset(txc->vxcr_u.txcd, 0,
+ txc->vxcr_ndesc * sizeof(struct vmxnet3_txcompdesc));
+}
+
+int
+vmxnet3_rxinit(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq)
+{
+ struct vmxnet3_rxring *rxr;
+ struct vmxnet3_comp_ring *rxc;
+ int i, populate, idx, error;
+
+ /* LRO and jumbo frame is not supported yet */
+ populate = 1;
+
+ for (i = 0; i < populate; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+ rxr->vxrxr_fill = 0;
+ rxr->vxrxr_gen = VMXNET3_INIT_GEN;
+ memset(rxr->vxrxr_rxd, 0,
+ rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
+
+ for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
+ error = vmxnet3_newbuf(sc, rxr);
+ if (error)
+ return (error);
+ }
+ }
+
+ for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
+ rxr = &rxq->vxrxq_cmd_ring[i];
+ rxr->vxrxr_fill = 0;
+ rxr->vxrxr_gen = 0;
+ memset(rxr->vxrxr_rxd, 0,
+ rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
+ }
+
+ rxc = &rxq->vxrxq_comp_ring;
+ rxc->vxcr_next = 0;
+ rxc->vxcr_gen = VMXNET3_INIT_GEN;
+ memset(rxc->vxcr_u.rxcd, 0,
+ rxc->vxcr_ndesc * sizeof(struct vmxnet3_rxcompdesc));
+
+ return (0);
+}
+
+int
+vmxnet3_reinit_queues(struct vmxnet3_softc *sc)
+{
+ device_t dev;
+ int q, error;
+ dev = sc->vmx_dev;
+
+ for (q = 0; q < sc->vmx_ntxqueues; q++)
+ vmxnet3_txinit(sc, &sc->vmx_txq[q]);
+
+ for (q = 0; q < sc->vmx_nrxqueues; q++) {
+ error = vmxnet3_rxinit(sc, &sc->vmx_rxq[q]);
+ if (error) {
+ device_printf(dev, "cannot populate Rx queue %d\n", q);
+ return (error);
+ }
+ }
+
+ return (0);
+}
+
+int
+vmxnet3_enable_device(struct vmxnet3_softc *sc)
+{
+ int q;
+
+ if (vmxnet3_read_cmd(sc, VMXNET3_CMD_ENABLE) != 0) {
+ device_printf(sc->vmx_dev, "device enable command failed!\n");
+ return (1);
+ }
+
+ /* Reset the Rx queue heads. */
+ for (q = 0; q < sc->vmx_nrxqueues; q++) {
+ vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH1(q), 0);
+ vmxnet3_write_bar0(sc, VMXNET3_BAR0_RXH2(q), 0);
+ }
+
+ return (0);
+}
+
+void
+vmxnet3_reinit_rxfilters(struct vmxnet3_softc *sc)
+{
+
+ vmxnet3_set_rxfilter(sc);
+
+ memset(sc->vmx_ds->vlan_filter, 0, sizeof(sc->vmx_ds->vlan_filter));
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_VLAN_FILTER);
+}
+
+int
+vmxnet3_reinit(struct vmxnet3_softc *sc)
+{
+
+ vmxnet3_set_lladdr(sc);
+ vmxnet3_reinit_shared_data(sc);
+
+ if (vmxnet3_reinit_queues(sc) != 0)
+ return (ENXIO);
+
+ if (vmxnet3_enable_device(sc) != 0)
+ return (ENXIO);
+
+ vmxnet3_reinit_rxfilters(sc);
+
+ return (0);
+}
+
+
+
+
+
+int
+vmxnet3_init_locked(struct vmxnet3_softc *sc)
+{
+ struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
+ int error;
+
+ if (ifp->if_flags & IFF_RUNNING)
+ return 0;
+
+ vmxnet3_stop_locked(sc);
+
+ error = vmxnet3_reinit(sc);
+ if (error) {
+ vmxnet3_stop_locked(sc);
+ return (error);
+ }
+
+ ifp->if_flags |= IFF_RUNNING;
+ vmxnet3_link_status(sc);
+
+ vmxnet3_enable_all_intrs(sc);
+ callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
+
+ return (0);
+}
+
+int
+vmxnet3_init(struct ifnet *ifp)
+{
+ struct vmxnet3_softc *sc = ifp->if_softc;
+ int error;
+
+ VMXNET3_CORE_LOCK(sc);
+ error = vmxnet3_init_locked(sc);
+ VMXNET3_CORE_UNLOCK(sc);
+
+ return (error);
+}
+
+int
+vmxnet3_txq_offload_ctx(struct vmxnet3_txqueue *txq, struct mbuf *m,
+ int *start, int *csum_start)
+{
+ struct ether_header *eh;
+ struct mbuf *mp;
+ int offset, csum_off, iphl, offp;
+ bool v4;
+
+ eh = mtod(m, struct ether_header *);
+ switch (htons(eh->ether_type)) {
+ case ETHERTYPE_IP:
+ case ETHERTYPE_IPV6:
+ offset = ETHER_HDR_LEN;
+ break;
+ case ETHERTYPE_VLAN:
+ offset = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
+ break;
+ default:
+ return (EINVAL);
+ }
+
+ if ((m->m_pkthdr.csum_flags &
+ (M_CSUM_TSOv4|M_CSUM_UDPv4|M_CSUM_TCPv4)) != 0) {
+ iphl = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
+ v4 = true;
+ } else {
+ iphl = M_CSUM_DATA_IPv6_HL(m->m_pkthdr.csum_data);
+ v4 = false;
+ }
+ *start = offset + iphl;
+
+ if (m->m_pkthdr.csum_flags &
+ (M_CSUM_TCPv4 | M_CSUM_TCPv4 | M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
+ csum_off = offsetof(struct tcphdr, th_sum);
+ } else {
+ csum_off = offsetof(struct udphdr, uh_sum);
+ }
+
+ *csum_start = *start + csum_off;
+ mp = m_pulldown(m, 0, *csum_start + 2, &offp);
+
+ if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
+ struct tcphdr *tcp;
+
+ txq->vxtxq_stats.vmtxs_tso++;
+ tcp = (void *)(mtod(mp, char *) + offp + *start);
+
+ if (v4) {
+ struct ip *ip;
+
+ ip = (void *)(mtod(mp, char *) + offp + offset);
+ tcp->th_sum = in_cksum_phdr(ip->ip_src.s_addr,
+ ip->ip_dst.s_addr, htons(IPPROTO_TCP));
+ } else {
+ struct ip6_hdr *ip6;
+
+ ip6 = (void *)(mtod(mp, char *) + offp + offset);
+ tcp->th_sum = in6_cksum_phdr(&ip6->ip6_src,
+ &ip6->ip6_dst, 0, htonl(IPPROTO_TCP));
+ }
+
+ /*
+ * For TSO, the size of the protocol header is also
+ * included in the descriptor header size.
+ */
+ *start += (tcp->th_off << 2);
+ } else
+ txq->vxtxq_stats.vmtxs_csum++;
+
+ return (0);
+}
+
+int
+vmxnet3_txq_load_mbuf(struct vmxnet3_txqueue *txq, struct mbuf **m0,
+ bus_dmamap_t dmap)
+{
+ struct mbuf *m;
+ bus_dma_tag_t tag;
+ int error;
- /* Link state change? */
- if (event & VMXNET3_EVENT_LINK)
- vmxnet3_link_state(sc);
+ m = *m0;
+ tag = txq->vxtxq_sc->vmx_dmat;
- /* Queue error? */
- if (event & (VMXNET3_EVENT_TQERROR | VMXNET3_EVENT_RQERROR)) {
- WRITE_CMD(sc, VMXNET3_CMD_GET_STATUS);
+ error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
+ if (error == 0 || error != EFBIG)
+ return (error);
+
+ m = m_defrag(m, M_NOWAIT);
+ if (m != NULL) {
+ *m0 = m;
+ error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
+ } else
+ error = ENOBUFS;
- ts = sc->sc_txq[0].ts;
- if (ts->stopped)
- printf("%s: TX error 0x%x\n", ifp->if_xname, ts->error);
- rs = sc->sc_rxq[0].rs;
- if (rs->stopped)
- printf("%s: RX error 0x%x\n", ifp->if_xname, rs->error);
- vmxnet3_reset(ifp);
- }
+ if (error) {
+ m_freem(*m0);
+ *m0 = NULL;
+ txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
+ } else
+ txq->vxtxq_sc->vmx_stats.vmst_defragged++;
- if (event & VMXNET3_EVENT_DIC)
- printf("%s: device implementation change event\n",
- ifp->if_xname);
- if (event & VMXNET3_EVENT_DEBUG)
- printf("%s: debug event\n", ifp->if_xname);
+ return (error);
}
void
-vmxnet3_txintr(struct vmxnet3_softc *sc, struct vmxnet3_txqueue *tq)
+vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
{
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
- struct vmxnet3_txcompdesc *txcd;
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- u_int sop;
- for (;;) {
- txcd = &comp_ring->txcd[comp_ring->next];
+ bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap);
+}
- if (le32toh((txcd->txc_word3 >> VMXNET3_TXC_GEN_S) &
- VMXNET3_TXC_GEN_M) != comp_ring->gen)
- break;
+int
+vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_txring *txr;
+ struct vmxnet3_txdesc *txd, *sop;
+ struct mbuf *m;
+ bus_dmamap_t dmap;
+ bus_dma_segment_t *segs;
+ struct m_tag *mtag;
+ int i, gen, start, csum_start, nsegs, error;
- comp_ring->next++;
- if (comp_ring->next == NTXCOMPDESC) {
- comp_ring->next = 0;
- comp_ring->gen ^= 1;
+ sc = txq->vxtxq_sc;
+ start = 0;
+ txd = NULL;
+ txr = &txq->vxtxq_cmd_ring;
+ dmap = txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_dmamap;
+
+ error = vmxnet3_txq_load_mbuf(txq, m0, dmap);
+ if (error)
+ return (error);
+
+ nsegs = dmap->dm_nsegs;
+ segs = dmap->dm_segs;
+
+ m = *m0;
+ KASSERT(m->m_flags & M_PKTHDR);
+ KASSERT(nsegs <= VMXNET3_TX_MAXSEGS);
+
+ if (vmxnet3_txring_avail(txr) < nsegs) {
+ txq->vxtxq_stats.vmtxs_full++;
+ vmxnet3_txq_unload_mbuf(txq, dmap);
+ return (ENOSPC);
+ } else if (m->m_pkthdr.csum_flags & VMXNET3_CSUM_ALL_OFFLOAD) {
+ error = vmxnet3_txq_offload_ctx(txq, m, &start, &csum_start);
+ if (error) {
+ txq->vxtxq_stats.vmtxs_offload_failed++;
+ vmxnet3_txq_unload_mbuf(txq, dmap);
+ m_freem(m);
+ *m0 = NULL;
+ return (error);
}
+ }
- sop = ring->next;
- if (ring->m[sop] == NULL)
- panic("vmxnet3_txintr");
- m_freem(ring->m[sop]);
- ring->m[sop] = NULL;
- bus_dmamap_unload(sc->sc_dmat, ring->dmap[sop]);
- ring->next = (le32toh((txcd->txc_word0 >>
- VMXNET3_TXC_EOPIDX_S) & VMXNET3_TXC_EOPIDX_M) + 1)
- % NTXDESC;
+ txr->vxtxr_txbuf[txr->vxtxr_head].vtxb_m = m;
+ sop = &txr->vxtxr_txd[txr->vxtxr_head];
+ gen = txr->vxtxr_gen ^ 1; /* Owned by cpu (yet) */
+
+ for (i = 0; i < nsegs; i++) {
+ txd = &txr->vxtxr_txd[txr->vxtxr_head];
+
+ txd->addr = segs[i].ds_addr;
+ txd->len = segs[i].ds_len;
+ txd->gen = gen;
+ txd->dtype = 0;
+ txd->offload_mode = VMXNET3_OM_NONE;
+ txd->offload_pos = 0;
+ txd->hlen = 0;
+ txd->eop = 0;
+ txd->compreq = 0;
+ txd->vtag_mode = 0;
+ txd->vtag = 0;
+
+ if (++txr->vxtxr_head == txr->vxtxr_ndesc) {
+ txr->vxtxr_head = 0;
+ txr->vxtxr_gen ^= 1;
+ }
+ gen = txr->vxtxr_gen;
+ }
+ txd->eop = 1;
+ txd->compreq = 1;
- ifp->if_flags &= ~IFF_OACTIVE;
+ if ((mtag = VLAN_OUTPUT_TAG(&sc->vmx_ethercom, m)) != NULL) {
+ sop->vtag_mode = 1;
+ sop->vtag = VLAN_TAG_VALUE(mtag);
+ }
+
+ if (m->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) {
+ sop->offload_mode = VMXNET3_OM_TSO;
+ sop->hlen = start;
+ sop->offload_pos = m->m_pkthdr.segsz;
+ } else if (m->m_pkthdr.csum_flags & (VMXNET3_CSUM_OFFLOAD |
+ VMXNET3_CSUM_OFFLOAD_IPV6)) {
+ sop->offload_mode = VMXNET3_OM_CSUM;
+ sop->hlen = start;
+ sop->offload_pos = csum_start;
+ }
+
+ /* Finally, change the ownership. */
+ vmxnet3_barrier(sc, VMXNET3_BARRIER_WR);
+ sop->gen ^= 1;
+
+ txq->vxtxq_ts->npending += nsegs;
+ if (txq->vxtxq_ts->npending >= txq->vxtxq_ts->intr_threshold) {
+ txq->vxtxq_ts->npending = 0;
+ vmxnet3_write_bar0(sc, VMXNET3_BAR0_TXH(txq->vxtxq_id),
+ txr->vxtxr_head);
}
- if (ring->head == ring->next)
- ifp->if_timer = 0;
- vmxnet3_start(ifp);
+
+ return (0);
}
void
-vmxnet3_rxintr(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rq)
+vmxnet3_start_locked(struct ifnet *ifp)
{
- struct vmxnet3_comp_ring *comp_ring = &rq->comp_ring;
- struct vmxnet3_rxring *ring;
- struct vmxnet3_rxdesc *rxd;
- struct vmxnet3_rxcompdesc *rxcd;
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- struct mbuf *m;
- int idx, len;
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_txqueue *txq;
+ struct vmxnet3_txring *txr;
+ struct mbuf *m_head;
+ int tx;
+
+ sc = ifp->if_softc;
+ txq = &sc->vmx_txq[0];
+ txr = &txq->vxtxq_cmd_ring;
+ tx = 0;
+
+ VMXNET3_TXQ_LOCK_ASSERT(txq);
+
+ if ((ifp->if_flags & IFF_RUNNING) == 0 ||
+ sc->vmx_link_active == 0)
+ return;
for (;;) {
- rxcd = &comp_ring->rxcd[comp_ring->next];
- if (le32toh((rxcd->rxc_word3 >> VMXNET3_RXC_GEN_S) &
- VMXNET3_RXC_GEN_M) != comp_ring->gen)
+ IFQ_POLL(&ifp->if_snd, m_head);
+ if (m_head == NULL)
break;
- comp_ring->next++;
- if (comp_ring->next == NRXCOMPDESC) {
- comp_ring->next = 0;
- comp_ring->gen ^= 1;
- }
-
- idx = le32toh((rxcd->rxc_word0 >> VMXNET3_RXC_IDX_S) &
- VMXNET3_RXC_IDX_M);
- if (le32toh((rxcd->rxc_word0 >> VMXNET3_RXC_QID_S) &
- VMXNET3_RXC_QID_M) < NRXQUEUE)
- ring = &rq->cmd_ring[0];
- else
- ring = &rq->cmd_ring[1];
- rxd = &ring->rxd[idx];
- len = le32toh((rxcd->rxc_word2 >> VMXNET3_RXC_LEN_S) &
- VMXNET3_RXC_LEN_M);
- m = ring->m[idx];
- ring->m[idx] = NULL;
- bus_dmamap_unload(sc->sc_dmat, ring->dmap[idx]);
+ if (vmxnet3_txring_avail(txr) < VMXNET3_TX_MAXSEGS)
+ break;
- if (m == NULL)
- panic("NULL mbuf");
+ IFQ_DEQUEUE(&ifp->if_snd, m_head);
+ if (m_head == NULL)
+ break;
- if (le32toh((rxd->rx_word2 >> VMXNET3_RX_BTYPE_S) &
- VMXNET3_RX_BTYPE_M) != VMXNET3_BTYPE_HEAD) {
- m_freem(m);
- goto skip_buffer;
- }
- if (le32toh(rxcd->rxc_word2 & VMXNET3_RXC_ERROR)) {
- ifp->if_ierrors++;
- m_freem(m);
- goto skip_buffer;
- }
- if (len < VMXNET3_MIN_MTU) {
- printf("%s: short packet (%d)\n", ifp->if_xname, len);
- m_freem(m);
- goto skip_buffer;
+ if (vmxnet3_txq_encap(txq, &m_head) != 0) {
+ if (m_head != NULL)
+ m_freem(m_head);
+ break;
}
- ifp->if_ipackets++;
- ifp->if_ibytes += len;
+ tx++;
+ bpf_mtap(ifp, m_head);
+ }
- vmxnet3_rx_csum(rxcd, m);
- m_set_rcvif(m, ifp);
- m->m_pkthdr.len = m->m_len = len;
- if (le32toh(rxcd->rxc_word2 & VMXNET3_RXC_VLAN)) {
- VLAN_INPUT_TAG(ifp, m,
- le32toh((rxcd->rxc_word2 >>
- VMXNET3_RXC_VLANTAG_S) & VMXNET3_RXC_VLANTAG_M),
- m_freem(m); goto skip_buffer);
- }
+ if (tx > 0)
+ txq->vxtxq_watchdog = VMXNET3_WATCHDOG_TIMEOUT;
+}
- bpf_mtap(ifp, m);
- if_percpuq_enqueue(ifp->if_percpuq, m);
+void
+vmxnet3_start(struct ifnet *ifp)
+{
+ struct vmxnet3_softc *sc;
+ struct vmxnet3_txqueue *txq;
-skip_buffer:
-#ifdef VMXNET3_STAT
- vmxstat.rxdone = idx;
-#endif
- if (rq->rs->update_rxhead) {
- u_int qid = le32toh((rxcd->rxc_word0 >>
- VMXNET3_RXC_QID_S) & VMXNET3_RXC_QID_M);
-
- idx = (idx + 1) % NRXDESC;
- if (qid < NRXQUEUE) {
- WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(qid), idx);
- } else {
- qid -= NRXQUEUE;
- WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(qid), idx);
- }
- }
- }
+ sc = ifp->if_softc;
+ txq = &sc->vmx_txq[0];
- /* XXX Should we (try to) allocate buffers for ring 2 too? */
- ring = &rq->cmd_ring[0];
- for (;;) {
- idx = ring->fill;
- if (ring->m[idx])
- return;
- if (vmxnet3_getbuf(sc, ring))
- return;
- }
+ VMXNET3_TXQ_LOCK(txq);
+ vmxnet3_start_locked(ifp);
+ VMXNET3_TXQ_UNLOCK(txq);
}
void
-vmxnet3_iff(struct vmxnet3_softc *sc)
+vmxnet3_set_rxfilter(struct vmxnet3_softc *sc)
{
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- struct ethercom *ec = &sc->sc_ethercom;
- struct vmxnet3_driver_shared *ds = sc->sc_ds;
+ struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
+ struct ethercom *ec = &sc->vmx_ethercom;
+ struct vmxnet3_driver_shared *ds = sc->vmx_ds;
struct ether_multi *enm;
struct ether_multistep step;
u_int mode;
@@ -821,7 +2815,7 @@ vmxnet3_iff(struct vmxnet3_softc *sc)
if (ISSET(ifp->if_flags, IFF_PROMISC) || ec->ec_multicnt > 682)
goto allmulti;
- p = sc->sc_mcast;
+ p = sc->vmx_mcast;
ETHER_FIRST_MULTI(step, ec, enm);
while (enm != NULL) {
if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
@@ -844,7 +2838,7 @@ vmxnet3_iff(struct vmxnet3_softc *sc)
if (ec->ec_multicnt > 0) {
SET(mode, VMXNET3_RXMODE_MCAST);
- ds->mcast_tablelen = p - sc->sc_mcast;
+ ds->mcast_tablelen = p - sc->vmx_mcast;
}
goto setit;
@@ -856,414 +2850,312 @@ allmulti:
SET(mode, VMXNET3_RXMODE_PROMISC);
setit:
- WRITE_CMD(sc, VMXNET3_CMD_SET_FILTER);
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_FILTER);
ds->rxmode = mode;
- WRITE_CMD(sc, VMXNET3_CMD_SET_RXMODE);
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_SET_RXMODE);
}
int
-vmxnet3_ifflags_cb(struct ethercom *ec)
+vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
{
+ struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
+
+ if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
+ return EINVAL;
- vmxnet3_iff((struct vmxnet3_softc *)ec->ec_if.if_softc);
+ if (ifp->if_flags & IFF_RUNNING) {
+ ifp->if_flags &= ~IFF_RUNNING;
+ vmxnet3_init_locked(sc);
+ }
return 0;
}
-
-void
-vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *rxcd, struct mbuf *m)
+int
+vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data)
{
- if (le32toh(rxcd->rxc_word0 & VMXNET3_RXC_NOCSUM))
- return;
+ struct vmxnet3_softc *sc = ifp->if_softc;
+ struct ifreq *ifr = (struct ifreq *)data;
+ int s, error = 0;
- if (rxcd->rxc_word3 & VMXNET3_RXC_IPV4) {
- m->m_pkthdr.csum_flags |= M_CSUM_IPv4;
- if ((rxcd->rxc_word3 & VMXNET3_RXC_IPSUM_OK) == 0)
- m->m_pkthdr.csum_flags |= M_CSUM_IPv4_BAD;
+ switch (cmd) {
+ case SIOCSIFMTU:
+ if (ifp->if_mtu != ifr->ifr_mtu) {
+ VMXNET3_CORE_LOCK(sc);
+ error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
+ VMXNET3_CORE_UNLOCK(sc);
+ }
+ break;
+ case SIOCSIFMEDIA:
+ case SIOCGIFMEDIA:
+ s = splnet();
+ error = ifmedia_ioctl(ifp, ifr, &sc->vmx_media, cmd);
+ splx(s);
+ break;
+ case SIOCGIFDATA:
+ for (int i = 0; i < sc->vmx_nrxqueues; i++) {
+ ifp->if_ipackets =
+ sc->vmx_rxq[i].vxrxq_stats.vmrxs_ipackets;
+ ifp->if_iqdrops =
+ sc->vmx_rxq[i].vxrxq_stats.vmrxs_iqdrops;
+ ifp->if_ierrors =
+ sc->vmx_rxq[i].vxrxq_stats.vmrxs_ierrors;
+ }
+ for (int i = 0; i < sc->vmx_ntxqueues; i++) {
+ ifp->if_opackets =
+ sc->vmx_txq[i].vxtxq_stats.vmtxs_opackets;
+ ifp->if_obytes =
+ sc->vmx_txq[i].vxtxq_stats.vmtxs_obytes;
+ ifp->if_omcasts =
+ sc->vmx_txq[i].vxtxq_stats.vmtxs_omcasts;
+ }
+ /* FALLTHROUGH */
+ default:
+ s = splnet();
+ error = ether_ioctl(ifp, cmd, data);
+ splx(s);
}
- if (rxcd->rxc_word3 & VMXNET3_RXC_FRAGMENT)
- return;
-
- if (rxcd->rxc_word3 & VMXNET3_RXC_TCP) {
- m->m_pkthdr.csum_flags |= M_CSUM_TCPv4;
- if ((rxcd->rxc_word3 & VMXNET3_RXC_CSUM_OK) == 0)
- m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
+ if (error == ENETRESET) {
+ VMXNET3_CORE_LOCK(sc);
+ if (ifp->if_flags & IFF_RUNNING)
+ vmxnet3_set_rxfilter(sc);
+ VMXNET3_CORE_UNLOCK(sc);
+ error = 0;
}
- if (rxcd->rxc_word3 & VMXNET3_RXC_UDP) {
- m->m_pkthdr.csum_flags |= M_CSUM_UDPv4;
- if ((rxcd->rxc_word3 & VMXNET3_RXC_CSUM_OK) == 0)
- m->m_pkthdr.csum_flags |= M_CSUM_TCP_UDP_BAD;
- }
+ return error;
}
int
-vmxnet3_getbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *ring)
+vmxnet3_ifflags_cb(struct ethercom *ec)
{
- int idx = ring->fill;
- struct vmxnet3_rxdesc *rxd = &ring->rxd[idx];
- struct mbuf *m;
- int btype;
+ struct vmxnet3_softc *sc;
- if (ring->m[idx])
- panic("vmxnet3_getbuf: buffer has mbuf");
+ sc = ec->ec_if.if_softc;
-#if 1
- /* XXX Don't allocate buffers for ring 2 for now. */
- if (ring->rid != 0)
- return -1;
- btype = VMXNET3_BTYPE_HEAD;
-#else
- if (ring->rid == 0)
- btype = VMXNET3_BTYPE_HEAD;
- else
- btype = VMXNET3_BTYPE_BODY;
-#endif
+ VMXNET3_CORE_LOCK(sc);
+ vmxnet3_set_rxfilter(sc);
+ VMXNET3_CORE_UNLOCK(sc);
- MGETHDR(m, M_DONTWAIT, MT_DATA);
- if (m == NULL)
- return -1;
+ return 0;
+}
- MCLGET(m, M_DONTWAIT);
- if ((m->m_flags & M_EXT) == 0) {
- m_freem(m);
- return -1;
- }
+int
+vmxnet3_watchdog(struct vmxnet3_txqueue *txq)
+{
+ struct vmxnet3_softc *sc;
- m->m_pkthdr.len = m->m_len = JUMBO_LEN;
- m_adj(m, ETHER_ALIGN);
- ring->m[idx] = m;
+ sc = txq->vxtxq_sc;
- if (bus_dmamap_load_mbuf(sc->sc_dmat, ring->dmap[idx], m,
- BUS_DMA_NOWAIT))
- panic("load mbuf");
- rxd->rx_addr = htole64(DMAADDR(ring->dmap[idx]));
- rxd->rx_word2 = htole32(((m->m_pkthdr.len & VMXNET3_RX_LEN_M) <<
- VMXNET3_RX_LEN_S) | ((btype & VMXNET3_RX_BTYPE_M) <<
- VMXNET3_RX_BTYPE_S) | ((ring->gen & VMXNET3_RX_GEN_M) <<
- VMXNET3_RX_GEN_S));
- idx++;
- if (idx == NRXDESC) {
- idx = 0;
- ring->gen ^= 1;
+ VMXNET3_TXQ_LOCK(txq);
+ if (txq->vxtxq_watchdog == 0 || --txq->vxtxq_watchdog) {
+ VMXNET3_TXQ_UNLOCK(txq);
+ return (0);
}
- ring->fill = idx;
-#ifdef VMXNET3_STAT
- vmxstat.rxfill = ring->fill;
-#endif
- return 0;
+ VMXNET3_TXQ_UNLOCK(txq);
+
+ device_printf(sc->vmx_dev, "watchdog timeout on queue %d\n",
+ txq->vxtxq_id);
+ return (1);
}
void
-vmxnet3_stop(struct ifnet *ifp, int disable)
+vmxnet3_refresh_host_stats(struct vmxnet3_softc *sc)
{
- struct vmxnet3_softc *sc = ifp->if_softc;
- int queue;
- if ((ifp->if_flags & IFF_RUNNING) == 0)
- return;
+ vmxnet3_write_cmd(sc, VMXNET3_CMD_GET_STATS);
+}
- vmxnet3_disable_all_intrs(sc);
- ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
- ifp->if_timer = 0;
+void
+vmxnet3_tick(void *xsc)
+{
+ struct vmxnet3_softc *sc;
+ struct ifnet *ifp;
+ int i, timedout;
- if (!disable)
- return;
+ sc = xsc;
+ ifp = &sc->vmx_ethercom.ec_if;
+ timedout = 0;
+
+ VMXNET3_CORE_LOCK(sc);
- WRITE_CMD(sc, VMXNET3_CMD_DISABLE);
+ vmxnet3_refresh_host_stats(sc);
- for (queue = 0; queue < NTXQUEUE; queue++)
- vmxnet3_txstop(sc, &sc->sc_txq[queue]);
- for (queue = 0; queue < NRXQUEUE; queue++)
- vmxnet3_rxstop(sc, &sc->sc_rxq[queue]);
+ for (i = 0; i < sc->vmx_ntxqueues; i++)
+ timedout |= vmxnet3_watchdog(&sc->vmx_txq[i]);
+
+ if (timedout != 0) {
+ ifp->if_flags &= ~IFF_RUNNING;
+ vmxnet3_init_locked(sc);
+ } else
+ callout_reset(&sc->vmx_tick, hz, vmxnet3_tick, sc);
+
+ VMXNET3_CORE_UNLOCK(sc);
}
void
-vmxnet3_reset(struct ifnet *ifp)
+vmxnet3_link_status(struct vmxnet3_softc *sc)
{
- struct vmxnet3_softc *sc = ifp->if_softc;
+ struct ifnet *ifp = &sc->vmx_ethercom.ec_if;
+ u_int x, link, speed;
+
+ x = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_LINK);
+ speed = x >> 16;
+ if (x & 1) {
+ sc->vmx_link_active = 1;
+ ifp->if_baudrate = IF_Mbps(speed);
+ link = LINK_STATE_UP;
+ } else {
+ sc->vmx_link_active = 0;
+ link = LINK_STATE_DOWN;
+ }
- vmxnet3_stop(ifp, 1);
- WRITE_CMD(sc, VMXNET3_CMD_RESET);
- vmxnet3_init(ifp);
+ if_link_state_change(ifp, link);
}
-int
-vmxnet3_init(struct ifnet *ifp)
+void
+vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct vmxnet3_softc *sc = ifp->if_softc;
- int queue;
-
- if (ifp->if_flags & IFF_RUNNING)
- return 0;
- ifp->if_flags |= IFF_RUNNING;
- ifp->if_flags &= ~IFF_OACTIVE;
+ vmxnet3_link_status(sc);
- for (queue = 0; queue < NTXQUEUE; queue++)
- vmxnet3_txinit(sc, &sc->sc_txq[queue]);
- for (queue = 0; queue < NRXQUEUE; queue++)
- vmxnet3_rxinit(sc, &sc->sc_rxq[queue]);
+ ifmr->ifm_status = IFM_AVALID;
+ ifmr->ifm_active = IFM_ETHER;
- WRITE_CMD(sc, VMXNET3_CMD_ENABLE);
- if (READ_BAR1(sc, VMXNET3_BAR1_CMD)) {
- printf("%s: failed to initialize\n", ifp->if_xname);
- vmxnet3_stop(ifp, 1);
- return EIO;
+ VMXNET3_CORE_LOCK(sc);
+ if (ifp->if_link_state != LINK_STATE_UP) {
+ VMXNET3_CORE_UNLOCK(sc);
+ return;
}
- for (queue = 0; queue < NRXQUEUE; queue++) {
- WRITE_BAR0(sc, VMXNET3_BAR0_RXH1(queue), 0);
- WRITE_BAR0(sc, VMXNET3_BAR0_RXH2(queue), 0);
- }
+ ifmr->ifm_status |= IFM_ACTIVE;
- vmxnet3_iff(sc);
- vmxnet3_enable_all_intrs(sc);
- vmxnet3_link_state(sc);
- return 0;
+ if (ifp->if_baudrate >= IF_Gbps(10ULL))
+ ifmr->ifm_active |= IFM_10G_T;
+ VMXNET3_CORE_UNLOCK(sc);
}
int
-vmxnet3_change_mtu(struct vmxnet3_softc *sc, int mtu)
+vmxnet3_media_change(struct ifnet *ifp)
{
- struct vmxnet3_driver_shared *ds = sc->sc_ds;
- struct ifnet *ifp = &sc->sc_ethercom.ec_if;
- int error;
-
- if (mtu < VMXNET3_MIN_MTU || mtu > VMXNET3_MAX_MTU)
- return EINVAL;
- vmxnet3_stop(ifp, 1);
- ifp->if_mtu = ds->mtu = mtu;
- error = vmxnet3_init(ifp);
- return error;
+ return 0;
}
-int
-vmxnet3_ioctl(struct ifnet *ifp, u_long cmd, void *data)
+void
+vmxnet3_set_lladdr(struct vmxnet3_softc *sc)
{
- struct vmxnet3_softc *sc = ifp->if_softc;
- struct ifreq *ifr = (struct ifreq *)data;
- int error = 0, s;
+ uint32_t ml, mh;
- s = splnet();
+ ml = sc->vmx_lladdr[0];
+ ml |= sc->vmx_lladdr[1] << 8;
+ ml |= sc->vmx_lladdr[2] << 16;
+ ml |= sc->vmx_lladdr[3] << 24;
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACL, ml);
+
+ mh = sc->vmx_lladdr[4];
+ mh |= sc->vmx_lladdr[5] << 8;
+ vmxnet3_write_bar1(sc, VMXNET3_BAR1_MACH, mh);
+}
- switch (cmd) {
- case SIOCSIFMTU:
- error = vmxnet3_change_mtu(sc, ifr->ifr_mtu);
- break;
- case SIOCSIFMEDIA:
- case SIOCGIFMEDIA:
- error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
- break;
- default:
- error = ether_ioctl(ifp, cmd, data);
- }
+void
+vmxnet3_get_lladdr(struct vmxnet3_softc *sc)
+{
+ uint32_t ml, mh;
- if (error == ENETRESET) {
- if (ifp->if_flags & IFF_RUNNING)
- vmxnet3_iff(sc);
- error = 0;
- }
+ ml = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACL);
+ mh = vmxnet3_read_cmd(sc, VMXNET3_CMD_GET_MACH);
- splx(s);
- return error;
+ sc->vmx_lladdr[0] = ml;
+ sc->vmx_lladdr[1] = ml >> 8;
+ sc->vmx_lladdr[2] = ml >> 16;
+ sc->vmx_lladdr[3] = ml >> 24;
+ sc->vmx_lladdr[4] = mh;
+ sc->vmx_lladdr[5] = mh >> 8;
}
void
-vmxnet3_start(struct ifnet *ifp)
+vmxnet3_enable_all_intrs(struct vmxnet3_softc *sc)
{
- struct vmxnet3_softc *sc = ifp->if_softc;
- struct vmxnet3_txqueue *tq = &sc->sc_txq[0];
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- struct mbuf *m;
- int n = 0;
-
- if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
- return;
-
- for (;;) {
- IFQ_POLL(&ifp->if_snd, m);
- if (m == NULL)
- break;
- if ((ring->next - ring->head - 1) % NTXDESC < NTXSEGS) {
- ifp->if_flags |= IFF_OACTIVE;
- break;
- }
+ int i;
- IFQ_DEQUEUE(&ifp->if_snd, m);
- if (vmxnet3_load_mbuf(sc, m) != 0) {
- ifp->if_oerrors++;
- continue;
- }
- bpf_mtap(ifp, m);
+ sc->vmx_ds->ictrl &= ~VMXNET3_ICTRL_DISABLE_ALL;
+ for (i = 0; i < sc->vmx_nintrs; i++)
+ vmxnet3_enable_intr(sc, i);
+}
- ifp->if_timer = 5;
- ifp->if_opackets++;
- n++;
- }
+void
+vmxnet3_disable_all_intrs(struct vmxnet3_softc *sc)
+{
+ int i;
- if (n > 0)
- WRITE_BAR0(sc, VMXNET3_BAR0_TXH(0), ring->head);
-#ifdef VMXNET3_STAT
- vmxstat.txhead = ring->head;
- vmxstat.txdone = ring->next;
- vmxstat.maxtxlen =
- max(vmxstat.maxtxlen, (ring->head - ring->next) % NTXDESC);
-#endif
+ sc->vmx_ds->ictrl |= VMXNET3_ICTRL_DISABLE_ALL;
+ for (i = 0; i < sc->vmx_nintrs; i++)
+ vmxnet3_disable_intr(sc, i);
}
+
int
-vmxnet3_load_mbuf(struct vmxnet3_softc *sc, struct mbuf *m)
+vmxnet3_dma_malloc(struct vmxnet3_softc *sc, bus_size_t size, bus_size_t align,
+ struct vmxnet3_dma_alloc *dma)
{
- struct vmxnet3_txqueue *tq = &sc->sc_txq[0];
- struct vmxnet3_txring *ring = &tq->cmd_ring;
- struct vmxnet3_txdesc *txd = NULL, *sop;
- struct mbuf *mp;
- struct m_tag *mtag;
- struct ip *ip;
- bus_dmamap_t map = ring->dmap[ring->head];
- u_int hlen = ETHER_HDR_LEN, csum_off = 0;
- int offp, gen, i;
-
-#if 0
- if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) {
- printf("%s: IP checksum offloading is not supported\n",
- sc->sc_dev.dv_xname);
- return -1;
- }
-#endif
- if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
- if (m->m_pkthdr.csum_flags & M_CSUM_TCPv4)
- csum_off = offsetof(struct tcphdr, th_sum);
- else
- csum_off = offsetof(struct udphdr, uh_sum);
-
- mp = m_pulldown(m, hlen, sizeof(*ip), &offp);
- if (mp == NULL)
- return (-1);
+ bus_dma_tag_t t = sc->vmx_dmat;
+ bus_dma_segment_t *segs = dma->dma_segs;
+ int n, error;
- ip = (struct ip *)(mp->m_data + offp);
- hlen += ip->ip_hl << 2;
+ memset(dma, 0, sizeof(*dma));
- mp = m_pulldown(m, 0, hlen + csum_off + 2, &offp);
- if (mp == NULL)
- return (-1);
+ error = bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT);
+ if (error) {
+ aprint_error_dev(sc->vmx_dev, "bus_dmamem_alloc failed: %d\n", error);
+ goto fail1;
}
+ KASSERT(n == 1);
- switch (bus_dmamap_load_mbuf(sc->sc_dmat, map, m, BUS_DMA_NOWAIT)) {
- case 0:
- break;
- case EFBIG:
- mp = m_defrag(m, M_DONTWAIT);
- if (mp != NULL) {
- m = mp;
- if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
- BUS_DMA_NOWAIT) == 0)
- break;
- }
- /* FALLTHROUGH */
- default:
- m_freem(m);
- return -1;
+ error = bus_dmamem_map(t, segs, 1, size, &dma->dma_vaddr, BUS_DMA_NOWAIT);
+ if (error) {
+ aprint_error_dev(sc->vmx_dev, "bus_dmamem_map failed: %d\n", error);
+ goto fail2;
}
- ring->m[ring->head] = m;
- sop = &ring->txd[ring->head];
- gen = ring->gen ^ 1; /* owned by cpu (yet) */
- for (i = 0; i < map->dm_nsegs; i++) {
- txd = &ring->txd[ring->head];
- txd->tx_addr = htole64(map->dm_segs[i].ds_addr);
- txd->tx_word2 = htole32(((map->dm_segs[i].ds_len &
- VMXNET3_TX_LEN_M) << VMXNET3_TX_LEN_S) |
- ((gen & VMXNET3_TX_GEN_M) << VMXNET3_TX_GEN_S));
- txd->tx_word3 = 0;
- ring->head++;
- if (ring->head == NTXDESC) {
- ring->head = 0;
- ring->gen ^= 1;
- }
- gen = ring->gen;
+ error = bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &dma->dma_map);
+ if (error) {
+ aprint_error_dev(sc->vmx_dev, "bus_dmamap_create failed: %d\n", error);
+ goto fail3;
}
- if (txd != NULL)
- txd->tx_word3 |= htole32(VMXNET3_TX_EOP | VMXNET3_TX_COMPREQ);
- if ((mtag = VLAN_OUTPUT_TAG(&sc->sc_ethercom, m)) != NULL) {
- sop->tx_word3 |= htole32(VMXNET3_TX_VTAG_MODE);
- sop->tx_word3 |= htole32((VLAN_TAG_VALUE(mtag) &
- VMXNET3_TX_VLANTAG_M) << VMXNET3_TX_VLANTAG_S);
- }
- if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4|M_CSUM_TCPv4)) {
- sop->tx_word2 |= htole32(((hlen + csum_off) &
- VMXNET3_TX_OP_M) << VMXNET3_TX_OP_S);
- sop->tx_word3 |= htole32(((hlen & VMXNET3_TX_HLEN_M) <<
- VMXNET3_TX_HLEN_S) | (VMXNET3_OM_CSUM << VMXNET3_TX_OM_S));
+ error = bus_dmamap_load(t, dma->dma_map, dma->dma_vaddr, size, NULL,
+ BUS_DMA_NOWAIT);
+ if (error) {
+ aprint_error_dev(sc->vmx_dev, "bus_dmamap_load failed: %d\n", error);
+ goto fail4;
}
- /* Change the ownership by flipping the "generation" bit */
- sop->tx_word2 ^= htole32(VMXNET3_TX_GEN_M << VMXNET3_TX_GEN_S);
+ memset(dma->dma_vaddr, 0, size);
+ dma->dma_paddr = DMAADDR(dma->dma_map);
+ dma->dma_size = size;
return (0);
+fail4:
+ bus_dmamap_destroy(t, dma->dma_map);
+fail3:
+ bus_dmamem_unmap(t, dma->dma_vaddr, size);
+fail2:
+ bus_dmamem_free(t, segs, 1);
+fail1:
+ return (error);
}
void
-vmxnet3_watchdog(struct ifnet *ifp)
-{
- int s;
-
- printf("%s: device timeout\n", ifp->if_xname);
- s = splnet();
- vmxnet3_stop(ifp, 1);
- vmxnet3_init(ifp);
- splx(s);
-}
-
-void
-vmxnet3_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
+vmxnet3_dma_free(struct vmxnet3_softc *sc, struct vmxnet3_dma_alloc *dma)
{
- struct vmxnet3_softc *sc = ifp->if_softc;
-
- vmxnet3_link_state(sc);
-
- ifmr->ifm_status = IFM_AVALID;
- ifmr->ifm_active = IFM_ETHER;
-
- if (ifp->if_link_state != LINK_STATE_UP)
- return;
-
- ifmr->ifm_status |= IFM_ACTIVE;
-
- if (ifp->if_baudrate >= IF_Gbps(10ULL))
- ifmr->ifm_active |= IFM_10G_T;
-}
+ bus_dma_tag_t t = sc->vmx_dmat;
-int
-vmxnet3_media_change(struct ifnet *ifp)
-{
- return 0;
-}
+ bus_dmamap_unload(t, dma->dma_map);
+ bus_dmamap_destroy(t, dma->dma_map);
+ bus_dmamem_unmap(t, dma->dma_vaddr, dma->dma_size);
+ bus_dmamem_free(t, dma->dma_segs, 1);
-void *
-vmxnet3_dma_allocmem(struct vmxnet3_softc *sc, u_int size, u_int align, bus_addr_t *pa)
-{
- bus_dma_tag_t t = sc->sc_dmat;
- bus_dma_segment_t segs[1];
- bus_dmamap_t map;
- void *va;
- int n;
-
- if (bus_dmamem_alloc(t, size, align, 0, segs, 1, &n, BUS_DMA_NOWAIT))
- return NULL;
- if (bus_dmamem_map(t, segs, 1, size, &va, BUS_DMA_NOWAIT))
- return NULL;
- if (bus_dmamap_create(t, size, 1, size, 0, BUS_DMA_NOWAIT, &map))
- return NULL;
- if (bus_dmamap_load(t, map, va, size, NULL, BUS_DMA_NOWAIT))
- return NULL;
- memset(va, 0, size);
- *pa = DMAADDR(map);
- bus_dmamap_unload(t, map);
- bus_dmamap_destroy(t, map);
- return va;
+ memset(dma, 0, sizeof(*dma));
}
Index: src/sys/arch/x86/pci/if_vmxreg.h
diff -u src/sys/arch/x86/pci/if_vmxreg.h:1.1 src/sys/arch/x86/pci/if_vmxreg.h:1.2
--- src/sys/arch/x86/pci/if_vmxreg.h:1.1 Tue Jun 10 01:42:39 2014
+++ src/sys/arch/x86/pci/if_vmxreg.h Fri Nov 25 05:29:54 2016
@@ -1,4 +1,4 @@
-/* $NetBSD: if_vmxreg.h,v 1.1 2014/06/10 01:42:39 hikaru Exp $ */
+/* $NetBSD: if_vmxreg.h,v 1.2 2016/11/25 05:29:54 hikaru Exp $ */
/* $OpenBSD: if_vmxreg.h,v 1.3 2013/08/28 10:19:19 reyk Exp $ */
/*
@@ -75,40 +75,35 @@ struct UPT1_RxStats {
#define VMXNET3_CMD_RESET 0xcafe0002 /* reset device */
#define VMXNET3_CMD_SET_RXMODE 0xcafe0003 /* set interface flags */
#define VMXNET3_CMD_SET_FILTER 0xcafe0004 /* set address filter */
+#define VMXNET3_CMD_VLAN_FILTER 0xcafe0005 /* set VLAN filter */
#define VMXNET3_CMD_GET_STATUS 0xf00d0000 /* get queue errors */
+#define VMXNET3_CMD_GET_STATS 0xf00d0001 /* get queue statistics */
#define VMXNET3_CMD_GET_LINK 0xf00d0002 /* get link status */
#define VMXNET3_CMD_GET_MACL 0xf00d0003
#define VMXNET3_CMD_GET_MACH 0xf00d0004
+#define VMXNET3_CMD_GET_INTRCFG 0xf00d0008 /* get interrupt config */
#define VMXNET3_DMADESC_ALIGN 128
+#define VMXNET3_INIT_GEN 1
/* All descriptors are in little-endian format. */
struct vmxnet3_txdesc {
- uint64_t tx_addr;
+ uint64_t addr;
- uint32_t tx_word2;
-#define VMXNET3_TX_LEN_M 0x00003fff
-#define VMXNET3_TX_LEN_S 0
-#define VMXNET3_TX_GEN_M 0x00000001 /* generation */
-#define VMXNET3_TX_GEN_S 14
-#define VMXNET3_TX_RES0 0x00008000
-#define VMXNET3_TX_DTYPE_M 0x00000001 /* descriptor type */
-#define VMXNET3_TX_DTYPE_S 16 /* descriptor type */
-#define VMXNET3_TX_RES1 0x00000002
-#define VMXNET3_TX_OP_M 0x00003fff /* offloading position */
-#define VMXNET3_TX_OP_S 18
-
- uint32_t tx_word3;
-#define VMXNET3_TX_HLEN_M 0x000003ff /* header len */
-#define VMXNET3_TX_HLEN_S 0
-#define VMXNET3_TX_OM_M 0x00000003 /* offloading mode */
-#define VMXNET3_TX_OM_S 10
-#define VMXNET3_TX_EOP 0x00001000 /* end of packet */
-#define VMXNET3_TX_COMPREQ 0x00002000 /* completion request */
-#define VMXNET3_TX_RES2 0x00004000
-#define VMXNET3_TX_VTAG_MODE 0x00008000 /* VLAN tag insertion mode */
-#define VMXNET3_TX_VLANTAG_M 0x0000ffff
-#define VMXNET3_TX_VLANTAG_S 16
+ uint32_t len:14;
+ uint32_t gen:1; /* Generation */
+ uint32_t pad1:1;
+ uint32_t dtype:1; /* Descriptor type */
+ uint32_t pad2:1;
+ uint32_t offload_pos:14; /* Offloading position */
+
+ uint32_t hlen:10; /* Header len */
+ uint32_t offload_mode:2; /* Offloading mode */
+ uint32_t eop:1; /* End of packet */
+ uint32_t compreq:1; /* Completion request */
+ uint32_t pad3:1;
+ uint32_t vtag_mode:1; /* VLAN tag insertion mode */
+ uint32_t vtag:16; /* VLAN tag */
} __packed;
/* offloading modes */
@@ -117,39 +112,27 @@ struct vmxnet3_txdesc {
#define VMXNET3_OM_TSO 3
struct vmxnet3_txcompdesc {
- uint32_t txc_word0;
-#define VMXNET3_TXC_EOPIDX_M 0x00000fff /* eop index in Tx ring */
-#define VMXNET3_TXC_EOPIDX_S 0
-#define VMXNET3_TXC_RES0_M 0x000fffff
-#define VMXNET3_TXC_RES0_S 12
-
- uint32_t txc_word1;
- uint32_t txc_word2;
-
- uint32_t txc_word3;
-#define VMXNET3_TXC_RES2_M 0x00ffffff
-#define VMXNET3_TXC_TYPE_M 0x0000007f
-#define VMXNET3_TXC_TYPE_S 24
-#define VMXNET3_TXC_GEN_M 0x00000001
-#define VMXNET3_TXC_GEN_S 31
+ uint32_t eop_idx:12; /* EOP index in Tx ring */
+ uint32_t pad1:20;
+
+ uint32_t pad2:32;
+ uint32_t pad3:32;
+
+ uint32_t rsvd:24;
+ uint32_t type:7;
+ uint32_t gen:1;
} __packed;
struct vmxnet3_rxdesc {
- uint64_t rx_addr;
+ uint64_t addr;
- uint32_t rx_word2;
-#define VMXNET3_RX_LEN_M 0x00003fff
-#define VMXNET3_RX_LEN_S 0
-#define VMXNET3_RX_BTYPE_M 0x00000001 /* buffer type */
-#define VMXNET3_RX_BTYPE_S 14
-#define VMXNET3_RX_DTYPE_M 0x00000001 /* descriptor type */
-#define VMXNET3_RX_DTYPE_S 15
-#define VMXNET3_RX_RES0_M 0x00007fff
-#define VMXNET3_RX_RES0_S 16
-#define VMXNET3_RX_GEN_M 0x00000001
-#define VMXNET3_RX_GEN_S 31
+ uint32_t len:14;
+ uint32_t btype:1; /* Buffer type */
+ uint32_t dtype:1; /* Descriptor type */
+ uint32_t rsvd:15;
+ uint32_t gen:1;
- uint32_t rx_word3;
+ uint32_t pad1:32;
} __packed;
/* buffer types */
@@ -157,48 +140,41 @@ struct vmxnet3_rxdesc {
#define VMXNET3_BTYPE_BODY 1 /* body only */
struct vmxnet3_rxcompdesc {
- uint32_t rxc_word0;
-#define VMXNET3_RXC_IDX_M 0x00000fff /* Rx descriptor index */
-#define VMXNET3_RXC_IDX_S 0
-#define VMXNET3_RXC_RES0_M 0x00000003
-#define VMXNET3_RXC_RES0_S 12
-#define VMXNET3_RXC_EOP 0x00004000 /* end of packet */
-#define VMXNET3_RXC_SOP 0x00008000 /* start of packet */
-#define VMXNET3_RXC_QID_M 0x000003ff
-#define VMXNET3_RXC_QID_S 16
-#define VMXNET3_RXC_RSSTYPE_M 0x0000000f
-#define VMXNET3_RXC_RSSTYPE_S 26
-#define VMXNET3_RXC_NOCSUM 0x40000000 /* no checksum calculated */
-#define VMXNET3_RXC_RES1 0x80000000
-
- uint32_t rxc_word1;
-#define VMXNET3_RXC_RSSHASH_M 0xffffffff /* RSS hash value */
-#define VMXNET3_RXC_RSSHASH_S 0
-
- uint32_t rxc_word2;
-#define VMXNET3_RXC_LEN_M 0x00003fff
-#define VMXNET3_RXC_LEN_S 0
-#define VMXNET3_RXC_ERROR 0x00004000
-#define VMXNET3_RXC_VLAN 0x00008000 /* 802.1Q VLAN frame */
-#define VMXNET3_RXC_VLANTAG_M 0x0000ffff /* VLAN tag */
-#define VMXNET3_RXC_VLANTAG_S 16
-
- uint32_t rxc_word3;
-#define VMXNET3_RXC_CSUM_M 0x0000ffff /* TCP/UDP checksum */
-#define VMXNET3_RXC_CSUM_S 16
-#define VMXNET3_RXC_CSUM_OK 0x00010000 /* TCP/UDP checksum ok */
-#define VMXNET3_RXC_UDP 0x00020000
-#define VMXNET3_RXC_TCP 0x00040000
-#define VMXNET3_RXC_IPSUM_OK 0x00080000 /* IP checksum ok */
-#define VMXNET3_RXC_IPV6 0x00100000
-#define VMXNET3_RXC_IPV4 0x00200000
-#define VMXNET3_RXC_FRAGMENT 0x00400000 /* IP fragment */
-#define VMXNET3_RXC_FCS 0x00800000 /* frame CRC correct */
-#define VMXNET3_RXC_TYPE_M 0x7f000000
-#define VMXNET3_RXC_GEN_M 0x00000001
-#define VMXNET3_RXC_GEN_S 31
+ uint32_t rxd_idx:12; /* Rx descriptor index */
+ uint32_t pad1:2;
+ uint32_t eop:1; /* End of packet */
+ uint32_t sop:1; /* Start of packet */
+ uint32_t qid:10;
+ uint32_t rss_type:4;
+ uint32_t no_csum:1; /* No checksum calculated */
+ uint32_t pad2:1;
+
+ uint32_t rss_hash:32; /* RSS hash value */
+
+ uint32_t len:14;
+ uint32_t error:1;
+ uint32_t vlan:1; /* 802.1Q VLAN frame */
+ uint32_t vtag:16; /* VLAN tag */
+
+ uint32_t csum:16;
+ uint32_t csum_ok:1; /* TCP/UDP checksum ok */
+ uint32_t udp:1;
+ uint32_t tcp:1;
+ uint32_t ipcsum_ok:1; /* IP checksum OK */
+ uint32_t ipv6:1;
+ uint32_t ipv4:1;
+ uint32_t fragment:1; /* IP fragment */
+ uint32_t fcs:1; /* Frame CRC correct */
+ uint32_t type:7;
+ uint32_t gen:1;
} __packed;
+#define VMXNET3_RCD_RSS_TYPE_NONE 0
+#define VMXNET3_RCD_RSS_TYPE_IPV4 1
+#define VMXNET3_RCD_RSS_TYPE_TCPIPV4 2
+#define VMXNET3_RCD_RSS_TYPE_IPV6 3
+#define VMXNET3_RCD_RSS_TYPE_TCPIPV6 4
+
#define VMXNET3_REV1_MAGIC 0xbabefee1
#define VMXNET3_GOS_UNKNOWN 0x00
@@ -214,7 +190,10 @@ struct vmxnet3_rxcompdesc {
#define VMXNET3_MAX_TX_QUEUES 8
#define VMXNET3_MAX_RX_QUEUES 16
#define VMXNET3_MAX_INTRS (VMXNET3_MAX_TX_QUEUES + VMXNET3_MAX_RX_QUEUES + 1)
-#define VMXNET3_NINTR 1
+
+#define VMXNET3_RX_INTR_INDEX 0
+#define VMXNET3_TX_INTR_INDEX 1
+#define VMXNET3_EV_INTR_INDEX 2
#define VMXNET3_ICTRL_DISABLE_ALL 0x01
@@ -233,6 +212,15 @@ struct vmxnet3_rxcompdesc {
#define VMXNET3_MAX_MTU 9000
#define VMXNET3_MIN_MTU 60
+#define VMXNET3_IMM_AUTO 0x00
+#define VMXNET3_IMM_ACTIVE 0x01
+#define VMXNET3_IMM_LAZY 0x02
+
+#define VMXNET3_IT_AUTO 0x00
+#define VMXNET3_IT_LEGACY 0x01
+#define VMXNET3_IT_MSI 0x02
+#define VMXNET3_IT_MSIX 0x03
+
struct vmxnet3_driver_shared {
uint32_t magic;
uint32_t pad1;
@@ -326,3 +314,25 @@ struct vmxnet3_rxq_shared {
uint8_t pad4[88];
} __packed;
+
+#define UPT1_RSS_HASH_TYPE_NONE 0x00
+#define UPT1_RSS_HASH_TYPE_IPV4 0x01
+#define UPT1_RSS_HASH_TYPE_TCP_IPV4 0x02
+#define UPT1_RSS_HASH_TYPE_IPV6 0x04
+#define UPT1_RSS_HASH_TYPE_TCP_IPV6 0x08
+
+#define UPT1_RSS_HASH_FUNC_NONE 0x00
+#define UPT1_RSS_HASH_FUNC_TOEPLITZ 0x01
+
+#define UPT1_RSS_MAX_KEY_SIZE 40
+#define UPT1_RSS_MAX_IND_TABLE_SIZE 128
+
+struct vmxnet3_rss_shared {
+ uint16_t hash_type;
+ uint16_t hash_func;
+ uint16_t hash_key_size;
+ uint16_t ind_table_size;
+ uint8_t hash_key[UPT1_RSS_MAX_KEY_SIZE];
+ uint8_t ind_table[UPT1_RSS_MAX_IND_TABLE_SIZE];
+} __packed;
+
