Module Name: src
Committed By: matt
Date: Thu Oct 4 00:14:25 UTC 2012
Modified Files:
src/sys/arch/arm/broadcom: bcm53xx_eth.c
Log Message:
Ethernet driver for BCM53XXX (not quite working).
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 src/sys/arch/arm/broadcom/bcm53xx_eth.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/broadcom/bcm53xx_eth.c
diff -u src/sys/arch/arm/broadcom/bcm53xx_eth.c:1.1 src/sys/arch/arm/broadcom/bcm53xx_eth.c:1.2
--- src/sys/arch/arm/broadcom/bcm53xx_eth.c:1.1 Sat Sep 1 00:04:44 2012
+++ src/sys/arch/arm/broadcom/bcm53xx_eth.c Thu Oct 4 00:14:24 2012
@@ -33,27 +33,86 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.1 2012/09/01 00:04:44 matt Exp $");
+__KERNEL_RCSID(1, "$NetBSD: bcm53xx_eth.c,v 1.2 2012/10/04 00:14:24 matt Exp $");
#include <sys/param.h>
+#include <sys/atomic.h>
#include <sys/bus.h>
#include <sys/device.h>
+#include <sys/ioctl.h>
#include <sys/intr.h>
+#include <sys/kmem.h>
#include <sys/mutex.h>
+#include <sys/socket.h>
#include <sys/systm.h>
#include <net/if.h>
#include <net/if_ether.h>
#include <net/if_media.h>
+#include <net/if_dl.h>
+
+#include <net/bpf.h>
+
#include <dev/mii/miivar.h>
#include <arm/broadcom/bcm53xx_reg.h>
#include <arm/broadcom/bcm53xx_var.h>
+#define BCMETH_RCVOFFSET 6
+#define BCMETH_MAXTXMBUFS 32
+#define BCMETH_NTXSEGS 30
+#define BCMETH_MAXRXMBUFS 255
+#define BCMETH_MINRXMBUFS 32
+#define BCMETH_NRXSEGS 1
+
static int bcmeth_ccb_match(device_t, cfdata_t, void *);
static void bcmeth_ccb_attach(device_t, device_t, void *);
+struct bcmeth_txqueue {
+ bus_dmamap_t txq_descmap;
+ struct gmac_txdb *txq_consumer;
+ struct gmac_txdb *txq_producer;
+ struct gmac_txdb *txq_first;
+ struct gmac_txdb *txq_last;
+ struct ifqueue txq_mbufs;
+ struct mbuf *txq_next;
+ size_t txq_free;
+ size_t txq_threshold;
+ size_t txq_lastintr;
+ bus_size_t txq_reg_xmtaddrlo;
+ bus_size_t txq_reg_xmtptr;
+ bus_size_t txq_reg_xmtctl;
+ bus_size_t txq_reg_xmtsts0;
+ bus_dma_segment_t txq_descmap_seg;
+};
+
+struct bcmeth_rxqueue {
+ bus_dmamap_t rxq_descmap;
+ struct gmac_rxdb *rxq_consumer;
+ struct gmac_rxdb *rxq_producer;
+ struct gmac_rxdb *rxq_first;
+ struct gmac_rxdb *rxq_last;
+ struct mbuf *rxq_mhead;
+ struct mbuf **rxq_mtail;
+ struct mbuf *rxq_mconsumer;
+ size_t rxq_inuse;
+ size_t rxq_threshold;
+ bus_size_t rxq_reg_rcvaddrlo;
+ bus_size_t rxq_reg_rcvptr;
+ bus_size_t rxq_reg_rcvctl;
+ bus_size_t rxq_reg_rcvsts0;
+ bus_dma_segment_t rxq_descmap_seg;
+};
+
+struct bcmeth_mapcache {
+ u_int dmc_nmaps;
+ u_int dmc_maxseg;
+ u_int dmc_maxmaps;
+ u_int dmc_maxmapsize;
+ bus_dmamap_t dmc_maps[0];
+};
+
struct bcmeth_softc {
device_t sc_dev;
bus_space_tag_t sc_bst;
@@ -62,13 +121,76 @@ struct bcmeth_softc {
kmutex_t *sc_lock;
kmutex_t *sc_hwlock;
struct ethercom sc_ec;
-#define sc_if sc_ec.ec_if;
- void *sc_sih;
+#define sc_if sc_ec.ec_if
+ struct ifmedia sc_media;
+ void *sc_soft_ih;
void *sc_ih;
+
+ struct bcmeth_rxqueue sc_rxq;
+ struct bcmeth_txqueue sc_txq;
+
+ uint32_t sc_maxfrm;
+ uint32_t sc_cmdcfg;
+ uint32_t sc_intmask;
+ volatile uint32_t sc_soft_flags;
+#define SOFT_RXINTR 0x01
+#define SOFT_RXUNDERFLOW 0x02
+#define SOFT_TXINTR 0x04
+#define SOFT_REINIT 0x08
+
+ struct evcnt sc_ev_intr;
+ struct evcnt sc_ev_soft_intr;
+ struct evcnt sc_ev_tx_stall;
+
+ struct ifqueue sc_rx_bufcache;
+ struct bcmeth_mapcache *sc_rx_mapcache;
+ struct bcmeth_mapcache *sc_tx_mapcache;
+
+ uint8_t sc_enaddr[ETHER_ADDR_LEN];
};
+static void bcmeth_ifstart(struct ifnet *);
+static void bcmeth_ifwatchdog(struct ifnet *);
+static int bcmeth_ifinit(struct ifnet *);
+static void bcmeth_ifstop(struct ifnet *, int);
+static int bcmeth_ifioctl(struct ifnet *, u_long, void *);
+
+static int bcmeth_mapcache_create(struct bcmeth_softc *,
+ struct bcmeth_mapcache **, size_t, size_t, size_t);
+static void bcmeth_mapcache_destroy(struct bcmeth_softc *,
+ struct bcmeth_mapcache *);
+static bus_dmamap_t bcmeth_mapcache_get(struct bcmeth_softc *,
+ struct bcmeth_mapcache *);
+static void bcmeth_mapcache_put(struct bcmeth_softc *,
+ struct bcmeth_mapcache *, bus_dmamap_t);
+
+static int bcmeth_txq_attach(struct bcmeth_softc *,
+ struct bcmeth_txqueue *, u_int);
+static void bcmeth_txq_purge(struct bcmeth_softc *,
+ struct bcmeth_txqueue *);
+static void bcmeth_txq_reset(struct bcmeth_softc *,
+ struct bcmeth_txqueue *);
+static bool bcmeth_txq_consume(struct bcmeth_softc *,
+ struct bcmeth_txqueue *);
+static bool bcmeth_txq_produce(struct bcmeth_softc *,
+ struct bcmeth_txqueue *, struct mbuf *m);
+static bool bcmeth_txq_active_p(struct bcmeth_softc *,
+ struct bcmeth_txqueue *);
+
+static int bcmeth_rxq_attach(struct bcmeth_softc *,
+ struct bcmeth_rxqueue *, u_int);
+static bool bcmeth_rxq_produce(struct bcmeth_softc *,
+ struct bcmeth_rxqueue *);
+static void bcmeth_rxq_purge(struct bcmeth_softc *,
+ struct bcmeth_rxqueue *, bool);
+static void bcmeth_rxq_reset(struct bcmeth_softc *,
+ struct bcmeth_rxqueue *);
+
static int bcmeth_intr(void *);
-static void bcmeth_softint(void *);
+static void bcmeth_soft_intr(void *);
+
+static int bcmeth_mediachange(struct ifnet *);
+static void bcmeth_mediastatus(struct ifnet *, struct ifmediareq *);
static inline uint32_t
bcmeth_read_4(struct bcmeth_softc *sc, bus_size_t o)
@@ -106,25 +228,76 @@ static void
bcmeth_ccb_attach(device_t parent, device_t self, void *aux)
{
struct bcmeth_softc * const sc = device_private(self);
+ struct ethercom * const ec = &sc->sc_ec;
+ struct ifnet * const ifp = &ec->ec_if;
struct bcmccb_attach_args * const ccbaa = aux;
const struct bcm_locators * const loc = &ccbaa->ccbaa_loc;
-
- sc->sc_dev = self;
- sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
- sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM);
+ const char * const xname = device_xname(self);
+ prop_dictionary_t dict = device_properties(self);
+ int error;
sc->sc_bst = ccbaa->ccbaa_ccb_bst;
sc->sc_dmat = ccbaa->ccbaa_dmat;
bus_space_subregion(sc->sc_bst, ccbaa->ccbaa_ccb_bsh,
loc->loc_offset, loc->loc_size, &sc->sc_bsh);
+ prop_data_t eaprop = prop_dictionary_get(dict, "mac-address");
+ if (eaprop == NULL) {
+ uint32_t mac0 = bcmeth_read_4(sc, UNIMAC_MAC_0);
+ uint32_t mac1 = bcmeth_read_4(sc, UNIMAC_MAC_1);
+ if ((mac0 == 0 && mac1 == 0) || (mac1 & 1)) {
+ aprint_error(": mac-address property is missing\n");
+ return;
+ }
+ sc->sc_enaddr[0] = (mac1 >> 0) & 0xff;
+ sc->sc_enaddr[1] = (mac1 >> 8) & 0xff;
+ sc->sc_enaddr[2] = (mac0 >> 0) & 0xff;
+ sc->sc_enaddr[3] = (mac0 >> 8) & 0xff;
+ sc->sc_enaddr[4] = (mac0 >> 16) & 0xff;
+ sc->sc_enaddr[5] = (mac0 >> 24) & 0xff;
+ } else {
+ KASSERT(prop_object_type(eaprop) == PROP_TYPE_DATA);
+ KASSERT(prop_data_size(eaprop) == ETHER_ADDR_LEN);
+ memcpy(sc->sc_enaddr, prop_data_data_nocopy(eaprop),
+ ETHER_ADDR_LEN);
+ }
+ sc->sc_dev = self;
+ sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
+ sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM);
+
bcmeth_write_4(sc, GMAC_INTMASK, 0); // disable interrupts
aprint_naive("\n");
aprint_normal(": Gigabit Ethernet Controller\n");
- sc->sc_sih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET,
- bcmeth_softint, sc);
+ error = bcmeth_rxq_attach(sc, &sc->sc_rxq, 0);
+ if (error) {
+ aprint_error(": failed to init rxq: %d\n", error);
+ return;
+ }
+
+ error = bcmeth_txq_attach(sc, &sc->sc_txq, 0);
+ if (error) {
+ aprint_error(": failed to init txq: %d\n", error);
+ return;
+ }
+
+ error = bcmeth_mapcache_create(sc, &sc->sc_rx_mapcache,
+ BCMETH_MAXRXMBUFS, MCLBYTES, BCMETH_NRXSEGS);
+ if (error) {
+ aprint_error(": failed to allocate rx dmamaps: %d\n", error);
+ return;
+ }
+
+ error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache,
+ BCMETH_MAXTXMBUFS, MCLBYTES, BCMETH_NTXSEGS);
+ if (error) {
+ aprint_error(": failed to allocate tx dmamaps: %d\n", error);
+ return;
+ }
+
+ sc->sc_soft_ih = softint_establish(SOFTINT_MPSAFE | SOFTINT_NET,
+ bcmeth_soft_intr, sc);
sc->sc_ih = intr_establish(loc->loc_intrs[0], IPL_VM, IST_LEVEL,
bcmeth_intr, sc);
@@ -136,29 +309,1290 @@ bcmeth_ccb_attach(device_t parent, devic
aprint_normal_dev(self, "interrupting on irq %d\n",
loc->loc_intrs[0]);
}
+
+ aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
+ ether_sprintf(sc->sc_enaddr));
+
+ /*
+ * Since each port in plugged into the switch/flow-accelerator,
+ * we hard code at Gige Full-Duplex with Flow Control enabled.
+ */
+ int ifmedia = IFM_ETHER|IFM_1000_T|IFM_FDX;
+ //ifmedia |= IFM_FLOW|IFM_ETH_TXPAUSE|IFM_ETH_RXPAUSE;
+ ifmedia_init(&sc->sc_media, IFM_IMASK, bcmeth_mediachange,
+ bcmeth_mediastatus);
+ ifmedia_add(&sc->sc_media, ifmedia, 0, NULL);
+ ifmedia_set(&sc->sc_media, ifmedia);
+
+ ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU;
+
+ strlcpy(ifp->if_xname, xname, IFNAMSIZ);
+ ifp->if_softc = sc;
+ ifp->if_baudrate = IF_Mbps(1000);
+ ifp->if_capabilities = 0;
+ ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+ ifp->if_ioctl = bcmeth_ifioctl;
+ ifp->if_start = bcmeth_ifstart;
+ ifp->if_watchdog = bcmeth_ifwatchdog;
+ ifp->if_init = bcmeth_ifinit;
+ ifp->if_stop = bcmeth_ifstop;
+ IFQ_SET_READY(&ifp->if_snd);
+
+ bcmeth_ifstop(ifp, true);
+
+ /*
+ * Attach the interface.
+ */
+ if_attach(ifp);
+ ether_ifattach(ifp, sc->sc_enaddr);
+
+ evcnt_attach_dynamic(&sc->sc_ev_intr, EVCNT_TYPE_INTR,
+ NULL, xname, "intr");
+ evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR,
+ NULL, xname, "soft intr");
+ evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC,
+ NULL, xname, "tx stalls");
+}
+
+static int
+bcmeth_mediachange(struct ifnet *ifp)
+{
+ //struct bcmeth_softc * const sc = ifp->if_softc;
+ return 0;
}
static void
-bcmeth_softint(void *arg)
+bcmeth_mediastatus(struct ifnet *ifp, struct ifmediareq *ifm)
{
- struct bcmeth_softc * const sc = arg;
+ //struct bcmeth_softc * const sc = ifp->if_softc;
- mutex_enter(sc->sc_lock);
- mutex_exit(sc->sc_lock);
+ ifm->ifm_status = IFM_AVALID | IFM_ACTIVE;
+ ifm->ifm_active = IFM_ETHER | IFM_FDX | IFM_1000_T;
+}
+
+static uint64_t
+bcmeth_macaddr_create(const uint8_t *enaddr)
+{
+ return (enaddr[2] << 0) // UNIMAC_MAC_0
+ | (enaddr[3] << 8) // UNIMAC_MAC_0
+ | (enaddr[4] << 16) // UNIMAC_MAC_0
+ | (enaddr[5] << 24) // UNIMAC_MAC_0
+ | ((uint64_t)enaddr[0] << 32) // UNIMAC_MAC_1
+ | ((uint64_t)enaddr[1] << 40); // UNIMAC_MAC_1
+}
+
+static int
+bcmeth_ifinit(struct ifnet *ifp)
+{
+ struct bcmeth_softc * const sc = ifp->if_softc;
+ int error = 0;
+
+ sc->sc_maxfrm = max(ifp->if_mtu + 32, MCLBYTES);
+ if (ifp->if_mtu > ETHERMTU_JUMBO)
+ return error;
+
+ KASSERT(ifp->if_flags & IFF_UP);
+
+ /*
+ * Stop the interface
+ */
+ bcmeth_ifstop(ifp, 0);
+
+ /*
+ * If our frame size has changed (or it's our first time through)
+ * destroy the existing transmit mapcache.
+ */
+ if (sc->sc_tx_mapcache != NULL
+ && sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) {
+ bcmeth_mapcache_destroy(sc, sc->sc_tx_mapcache);
+ sc->sc_tx_mapcache = NULL;
+ }
+
+ if (sc->sc_tx_mapcache == NULL) {
+ error = bcmeth_mapcache_create(sc, &sc->sc_tx_mapcache,
+ BCMETH_MAXTXMBUFS, sc->sc_maxfrm, BCMETH_NTXSEGS);
+ if (error)
+ return error;
+ }
+
+ sc->sc_cmdcfg = NO_LENGTH_CHECK | PAUSE_IGNORE
+ | __SHIFTIN(ETH_SPEED_1000, ETH_SPEED)
+ | RX_ENA | TX_ENA;
+
+ if (ifp->if_flags & IFF_PROMISC) {
+ sc->sc_cmdcfg |= PROMISC_EN;
+ } else {
+ sc->sc_cmdcfg &= ~PROMISC_EN;
+ }
+
+ const uint64_t macstnaddr =
+ bcmeth_macaddr_create(CLLADDR(ifp->if_sadl));
+
+ sc->sc_intmask = DESCPROTOERR|DATAERR|DESCERR;
+
+#if 1
+ /* 5. Load RCVADDR_LO with new pointer */
+ bcmeth_rxq_reset(sc, &sc->sc_rxq);
+
+ bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl, BCMETH_RCVOFFSET
+ | RCVCTL_PARITY_DIS
+ | RCVCTL_OFLOW_CONTINUE
+ | __SHIFTIN(4, RCVCTL_BURSTLEN));
+#endif
+
+ /* 6. Load XMTADDR_LO with new pointer */
+ bcmeth_txq_reset(sc, &sc->sc_txq);
+
+ bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl, XMTCTL_DMA_ACT_INDEX
+ | XMTCTL_PARITY_DIS
+ | __SHIFTIN(4, XMTCTL_BURSTLEN));
+
+ /* 7. Setup other UNIMAC registers */
+ bcmeth_write_4(sc, UNIMAC_FRAME_LEN, sc->sc_maxfrm);
+ bcmeth_write_4(sc, UNIMAC_MAC_0, (uint32_t)(macstnaddr >> 0));
+ bcmeth_write_4(sc, UNIMAC_MAC_1, (uint32_t)(macstnaddr >> 32));
+ bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, sc->sc_cmdcfg);
+
+ uint32_t devctl = bcmeth_read_4(sc, GMAC_DEVCONTROL);
+ devctl |= RGMII_LINK_STATUS_SEL | NWAY_AUTO_POLL_EN | TXARB_STRICT_MODE;
+ devctl &= ~FLOW_CTRL_MODE;
+ devctl &= ~MIB_RD_RESET_EN;
+ devctl &= ~RXQ_OVERFLOW_CTRL_SEL;
+ devctl &= ~CPU_FLOW_CTRL_ON;
+ bcmeth_write_4(sc, GMAC_DEVCONTROL, devctl);
+
+ /* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */
+ sc->sc_intmask |= XMTINT_0|XMTUF;
+ bcmeth_write_4(sc, sc->sc_txq.txq_reg_xmtctl,
+ bcmeth_read_4(sc, sc->sc_txq.txq_reg_xmtctl) | XMTCTL_ENABLE);
+
+
+#if 1
+ /* 12. Enable receive queues in RQUEUE, */
+ sc->sc_intmask |= RCVINT|RCVDESCUF|RCVFIFOOF;
+ bcmeth_write_4(sc, sc->sc_rxq.rxq_reg_rcvctl,
+ bcmeth_read_4(sc, sc->sc_rxq.rxq_reg_rcvctl) | RCVCTL_ENABLE);
+
+ bcmeth_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */
+#endif
+
+ sc->sc_soft_flags = 0;
+
+ bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask);
+
+ ifp->if_flags |= IFF_RUNNING;
+
+ return error;
+}
+
+static void
+bcmeth_ifstop(struct ifnet *ifp, int disable)
+{
+ struct bcmeth_softc * const sc = ifp->if_softc;
+ struct bcmeth_txqueue * const txq = &sc->sc_txq;
+ struct bcmeth_rxqueue * const rxq = &sc->sc_rxq;
+
+ KASSERT(!cpu_intr_p());
+
+ sc->sc_soft_flags = 0;
+
+ /* Disable Rx processing */
+ bcmeth_write_4(sc, rxq->rxq_reg_rcvctl,
+ bcmeth_read_4(sc, rxq->rxq_reg_rcvctl) & ~RCVCTL_ENABLE);
+
+ /* Disable Tx processing */
+ bcmeth_write_4(sc, txq->txq_reg_xmtctl,
+ bcmeth_read_4(sc, txq->txq_reg_xmtctl) & ~XMTCTL_ENABLE);
+
+ /* Disable all interrupts */
+ bcmeth_write_4(sc, GMAC_INTMASK, 0);
+
+ for (;;) {
+ uint32_t tx0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0);
+ uint32_t rx0 = bcmeth_read_4(sc, rxq->rxq_reg_rcvsts0);
+ if (__SHIFTOUT(tx0, XMTSTATE) == XMTSTATE_DIS
+ && __SHIFTOUT(rx0, RCVSTATE) == RCVSTATE_DIS)
+ break;
+ delay(50);
+ }
+ /*
+ * Now reset the controller.
+ *
+ * 3. Set SW_RESET bit in UNIMAC_COMMAND_CONFIG register
+ * 4. Clear SW_RESET bit in UNIMAC_COMMAND_CONFIG register
+ */
+ bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, SW_RESET);
+ bcmeth_write_4(sc, GMAC_INTSTATUS, ~0);
+ sc->sc_intmask = 0;
+ ifp->if_flags &= ~IFF_RUNNING;
+
+ /*
+ * Let's consume any remaining transmitted packets. And if we are
+ * disabling the interface, purge ourselves of any untransmitted
+ * packets. But don't consume any received packets, just drop them.
+ * If we aren't disabling the interface, save the mbufs in the
+ * receive queue for reuse.
+ */
+ bcmeth_rxq_purge(sc, &sc->sc_rxq, disable);
+ bcmeth_txq_consume(sc, &sc->sc_txq);
+ if (disable) {
+ bcmeth_txq_purge(sc, &sc->sc_txq);
+ IF_PURGE(&ifp->if_snd);
+ }
+
+ bcmeth_write_4(sc, UNIMAC_COMMAND_CONFIG, 0);
+}
+
+static void
+bcmeth_ifwatchdog(struct ifnet *ifp)
+{
+}
+
+static int
+bcmeth_ifioctl(struct ifnet *ifp, u_long cmd, void *data)
+{
+ struct bcmeth_softc *sc = ifp->if_softc;
+ struct ifreq * const ifr = data;
+ const int s = splnet();
+ int error;
+
+ switch (cmd) {
+ case SIOCSIFMEDIA:
+ case SIOCGIFMEDIA:
+ error = ifmedia_ioctl(ifp, ifr, &sc->sc_media, cmd);
+ break;
+
+ default:
+ error = ether_ioctl(ifp, cmd, data);
+ if (error != ENETRESET)
+ break;
+
+ if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) {
+ error = 0;
+ break;
+ }
+ error = bcmeth_ifinit(ifp);
+ break;
+ }
+
+ splx(s);
+ return error;
+}
+
+static void
+bcmeth_rxq_desc_presync(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq,
+ struct gmac_rxdb *rxdb,
+ size_t count)
+{
+ bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap,
+ (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb),
+ BUS_DMASYNC_PREWRITE);
+}
+
+static void
+bcmeth_rxq_desc_postsync(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq,
+ struct gmac_rxdb *rxdb,
+ size_t count)
+{
+ bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap,
+ (rxdb - rxq->rxq_first) * sizeof(*rxdb), count * sizeof(*rxdb),
+ BUS_DMASYNC_POSTWRITE);
+}
+
+static void
+bcmeth_txq_desc_presync(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ struct gmac_txdb *txdb,
+ size_t count)
+{
+ bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap,
+ (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb),
+ BUS_DMASYNC_PREWRITE);
+}
+
+static void
+bcmeth_txq_desc_postsync(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ struct gmac_txdb *txdb,
+ size_t count)
+{
+ bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap,
+ (txdb - txq->txq_first) * sizeof(*txdb), count * sizeof(*txdb),
+ BUS_DMASYNC_POSTWRITE);
+}
+
+static bus_dmamap_t
+bcmeth_mapcache_get(
+ struct bcmeth_softc *sc,
+ struct bcmeth_mapcache *dmc)
+{
+ KASSERT(dmc->dmc_nmaps > 0);
+ KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL);
+ return dmc->dmc_maps[--dmc->dmc_nmaps];
+}
+
+static void
+bcmeth_mapcache_put(
+ struct bcmeth_softc *sc,
+ struct bcmeth_mapcache *dmc,
+ bus_dmamap_t map)
+{
+ KASSERT(map != NULL);
+ KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps);
+ dmc->dmc_maps[dmc->dmc_nmaps++] = map;
+}
+
+static void
+bcmeth_mapcache_destroy(
+ struct bcmeth_softc *sc,
+ struct bcmeth_mapcache *dmc)
+{
+ const size_t dmc_size =
+ offsetof(struct bcmeth_mapcache, dmc_maps[dmc->dmc_maxmaps]);
+
+ for (u_int i = 0; i < dmc->dmc_maxmaps; i++) {
+ bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]);
+ }
+ kmem_intr_free(dmc, dmc_size);
+}
+
+static int
+bcmeth_mapcache_create(
+ struct bcmeth_softc *sc,
+ struct bcmeth_mapcache **dmc_p,
+ size_t maxmaps,
+ size_t maxmapsize,
+ size_t maxseg)
+{
+ const size_t dmc_size =
+ offsetof(struct bcmeth_mapcache, dmc_maps[maxmaps]);
+ struct bcmeth_mapcache * const dmc =
+ kmem_intr_zalloc(dmc_size, KM_NOSLEEP);
+
+ dmc->dmc_maxmaps = maxmaps;
+ dmc->dmc_nmaps = maxmaps;
+ dmc->dmc_maxmapsize = maxmapsize;
+ dmc->dmc_maxseg = maxseg;
+
+ for (u_int i = 0; i < maxmaps; i++) {
+ int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize,
+ dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0,
+ BUS_DMA_WAITOK|BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]);
+ if (error) {
+ aprint_error_dev(sc->sc_dev,
+ "failed to creat dma map cache "
+ "entry %u of %zu: %d\n",
+ i, maxmaps, error);
+ while (i-- > 0) {
+ bus_dmamap_destroy(sc->sc_dmat,
+ dmc->dmc_maps[i]);
+ }
+ kmem_intr_free(dmc, dmc_size);
+ return error;
+ }
+ KASSERT(dmc->dmc_maps[i] != NULL);
+ }
+
+ *dmc_p = dmc;
+
+ return 0;
+}
+
+#if 0
+static void
+bcmeth_dmamem_free(
+ bus_dma_tag_t dmat,
+ size_t map_size,
+ bus_dma_segment_t *seg,
+ bus_dmamap_t map,
+ void *kvap)
+{
+ bus_dmamap_destroy(dmat, map);
+ bus_dmamem_unmap(dmat, kvap, map_size);
+ bus_dmamem_free(dmat, seg, 1);
+}
+#endif
+
+static int
+bcmeth_dmamem_alloc(
+ bus_dma_tag_t dmat,
+ size_t map_size,
+ bus_dma_segment_t *seg,
+ bus_dmamap_t *map,
+ void **kvap)
+{
+ int error;
+ int nseg;
+
+ *kvap = NULL;
+ *map = NULL;
+
+ error = bus_dmamem_alloc(dmat, map_size, PAGE_SIZE, 0,
+ seg, 1, &nseg, 0);
+ if (error)
+ return error;
+
+ KASSERT(nseg == 1);
+
+ error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap,
+ BUS_DMA_COHERENT);
+ if (error == 0) {
+ error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0,
+ map);
+ if (error == 0) {
+ error = bus_dmamap_load(dmat, *map, *kvap, map_size,
+ NULL, 0);
+ if (error == 0)
+ return 0;
+ bus_dmamap_destroy(dmat, *map);
+ *map = NULL;
+ }
+ bus_dmamem_unmap(dmat, *kvap, map_size);
+ *kvap = NULL;
+ }
+ bus_dmamem_free(dmat, seg, nseg);
+ return 0;
+}
+
+static struct mbuf *
+bcmeth_rx_buf_alloc(
+ struct bcmeth_softc *sc)
+{
+ struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA);
+ if (m == NULL) {
+ printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr");
+ return NULL;
+ }
+ MCLGET(m, M_DONTWAIT);
+ if ((m->m_flags & M_EXT) == 0) {
+ printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET");
+ m_freem(m);
+ return NULL;
+ }
+ m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
+
+ bus_dmamap_t map = bcmeth_mapcache_get(sc, sc->sc_rx_mapcache);
+ if (map == NULL) {
+ printf("%s:%d: %s\n", __func__, __LINE__, "map get");
+ m_freem(m);
+ return NULL;
+ }
+ M_SETCTX(m, map);
+ m->m_len = m->m_pkthdr.len = MCLBYTES;
+ int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
+ BUS_DMA_READ|BUS_DMA_NOWAIT);
+ if (error) {
+ aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n",
+ error);
+ M_SETCTX(m, NULL);
+ m_freem(m);
+ bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map);
+ return NULL;
+ }
+ KASSERT(map->dm_mapsize == MCLBYTES);
+ bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
+ BUS_DMASYNC_PREREAD);
+
+ return m;
+}
+
+static void
+bcmeth_rx_map_unload(
+ struct bcmeth_softc *sc,
+ struct mbuf *m)
+{
+ KASSERT(m);
+ for (; m != NULL; m = m->m_next) {
+ bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
+ KASSERT(map);
+ KASSERT(map->dm_mapsize == MCLBYTES);
+ bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len,
+ BUS_DMASYNC_POSTREAD);
+ bus_dmamap_unload(sc->sc_dmat, map);
+ bcmeth_mapcache_put(sc, sc->sc_rx_mapcache, map);
+ M_SETCTX(m, NULL);
+ }
+}
+
+static bool
+bcmeth_rxq_produce(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq)
+{
+ struct gmac_rxdb *producer = rxq->rxq_producer;
+#if 0
+ size_t inuse = rxq->rxq_inuse;
+#endif
+ while (rxq->rxq_inuse < rxq->rxq_threshold) {
+ struct mbuf *m;
+ IF_DEQUEUE(&sc->sc_rx_bufcache, m);
+ if (m == NULL) {
+ m = bcmeth_rx_buf_alloc(sc);
+ if (m == NULL) {
+ printf("%s: bcmeth_rx_buf_alloc failed\n", __func__);
+ break;
+ }
+ }
+ bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
+ KASSERT(map);
+
+ producer->rxdb_buflen = MCLBYTES;
+ producer->rxdb_addrlo = map->dm_segs[0].ds_addr;
+ producer->rxdb_flags &= RXDB_FLAG_ET;
+ *rxq->rxq_mtail = m;
+ rxq->rxq_mtail = &m->m_next;
+ m->m_len = MCLBYTES;
+ m->m_next = NULL;
+ rxq->rxq_inuse++;
+ if (++producer == rxq->rxq_last) {
+ membar_producer();
+ bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer,
+ rxq->rxq_last - rxq->rxq_producer);
+ producer = rxq->rxq_producer = rxq->rxq_first;
+ }
+ }
+ if (producer != rxq->rxq_producer) {
+ membar_producer();
+ bcmeth_rxq_desc_presync(sc, rxq, rxq->rxq_producer,
+ producer - rxq->rxq_producer);
+ rxq->rxq_producer = producer;
+ bcmeth_write_4(sc, rxq->rxq_reg_rcvptr,
+ rxq->rxq_descmap->dm_segs[0].ds_addr
+ + ((uintptr_t)rxq->rxq_producer & RCVPTR));
+ }
+ return true;
+}
+
+static void
+bcmeth_rx_input(
+ struct bcmeth_softc *sc,
+ struct mbuf *m,
+ uint32_t rxdb_flags)
+{
+ struct ifnet * const ifp = &sc->sc_if;
+
+ bcmeth_rx_map_unload(sc, m);
+
+ m_adj(m, BCMETH_RCVOFFSET);
+
+ switch (__SHIFTOUT(rxdb_flags, RXSTS_PKTTYPE)) {
+ case RXSTS_PKTTYPE_UC:
+ break;
+ case RXSTS_PKTTYPE_MC:
+ m->m_flags |= M_MCAST;
+ break;
+ case RXSTS_PKTTYPE_BC:
+ m->m_flags |= M_BCAST|M_MCAST;
+ break;
+ }
+ if (sc->sc_cmdcfg & PROMISC_EN)
+ m->m_flags |= M_PROMISC;
+ m->m_flags |= M_HASFCS;
+ m->m_pkthdr.rcvif = ifp;
+
+ ifp->if_ipackets++;
+ ifp->if_ibytes += m->m_pkthdr.len;
+
+ /*
+ * Let's give it to the network subsystm to deal with.
+ */
+ int s = splnet();
+ bpf_mtap(ifp, m);
+ (*ifp->if_input)(ifp, m);
+ splx(s);
+}
+
+static void
+bcmeth_rxq_consume(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq)
+{
+ struct ifnet * const ifp = &sc->sc_if;
+ struct gmac_rxdb *consumer = rxq->rxq_consumer;
+ size_t rxconsumed = 0;
+
+ for (;;) {
+ if (consumer == rxq->rxq_producer) {
+ rxq->rxq_consumer = consumer;
+ rxq->rxq_inuse -= rxconsumed;
+ KASSERT(rxq->rxq_inuse == 0);
+ return;
+ }
+
+ uint32_t rcvsts0 = bcmeth_read_4(sc, GMAC_RCVSTATUS0);
+ uint32_t currdscr = __SHIFTOUT(rcvsts0, RCV_CURRDSCR);
+ if (consumer == rxq->rxq_first + currdscr) {
+ rxq->rxq_consumer = consumer;
+ rxq->rxq_inuse -= rxconsumed;
+ return;
+ }
+ bcmeth_rxq_desc_postsync(sc, rxq, consumer, 1);
+
+ /*
+ * We own this packet again. Copy the rxsts word from it.
+ */
+ rxconsumed++;
+ uint32_t rxsts;
+ KASSERT(rxq->rxq_mhead != NULL);
+ bus_dmamap_t map = M_GETCTX(rxq->rxq_mhead, bus_dmamap_t);
+ bus_dmamap_sync(sc->sc_dmat, map, 0, arm_dcache_align,
+ BUS_DMASYNC_POSTREAD);
+ memcpy(&rxsts, rxq->rxq_mhead->m_data, 4);
+
+ /*
+ * Get the count of descriptors. Fetch the correct number
+ * of mbufs.
+ */
+ size_t desc_count = __SHIFTOUT(rxsts, RXSTS_DESC_COUNT) + 1;
+ struct mbuf *m = rxq->rxq_mhead;
+ struct mbuf *m_last = m;
+ for (size_t i = 1; i < desc_count; i++) {
+ if (++consumer == rxq->rxq_last) {
+ consumer = rxq->rxq_first;
+ }
+ KASSERT(consumer != rxq->rxq_first + currdscr);
+ m_last = m_last->m_next;
+ }
+
+ /*
+ * Now remove it/them from the list of enqueued mbufs.
+ */
+ if ((rxq->rxq_mhead = m_last->m_next) == NULL)
+ rxq->rxq_mtail = &rxq->rxq_mhead;
+ m_last->m_next = NULL;
+
+ if (rxsts & (/*RXSTS_CRC_ERROR|*/RXSTS_OVERSIZED|RXSTS_PKT_OVERFLOW)) {
+ aprint_error_dev(sc->sc_dev, "[%zu]: count=%zu rxsts=%#x\n",
+ consumer - rxq->rxq_first, desc_count, rxsts);
+ /*
+ * We encountered an error, take the mbufs and add them
+ * to the rx bufcache so we can quickly reuse them.
+ */
+ ifp->if_ierrors++;
+ do {
+ struct mbuf *m0 = m->m_next;
+ m->m_next = NULL;
+ IF_ENQUEUE(&sc->sc_rx_bufcache, m);
+ m = m0;
+ } while (m);
+ } else {
+ uint32_t framelen = __SHIFTOUT(rxsts, RXSTS_FRAMELEN);
+ framelen += BCMETH_RCVOFFSET;
+ m->m_pkthdr.len = framelen;
+ if (desc_count == 1) {
+ KASSERT(framelen <= MCLBYTES);
+ m->m_len = framelen;
+ } else {
+ m_last->m_len = framelen & (MCLBYTES - 1);
+ }
+ bcmeth_rx_input(sc, m, rxsts);
+ }
+
+ /*
+ * Wrap at the last entry!
+ */
+ if (++consumer == rxq->rxq_last) {
+ KASSERT(consumer[-1].rxdb_flags & RXDB_FLAG_ET);
+ consumer = rxq->rxq_first;
+ }
+ }
+}
+
+static void
+bcmeth_rxq_purge(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq,
+ bool discard)
+{
+ struct mbuf *m;
+
+ if ((m = rxq->rxq_mhead) != NULL) {
+ if (discard) {
+ bcmeth_rx_map_unload(sc, m);
+ m_freem(m);
+ } else {
+ while (m != NULL) {
+ struct mbuf *m0 = m->m_next;
+ m->m_next = NULL;
+ IF_ENQUEUE(&sc->sc_rx_bufcache, m);
+ m = m0;
+ }
+ }
+
+ }
+
+ rxq->rxq_mhead = NULL;
+ rxq->rxq_mtail = &rxq->rxq_mhead;
+ rxq->rxq_inuse = 0;
+}
+
+static void
+bcmeth_rxq_reset(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq)
+{
+ /*
+ * Reset the producer consumer indexes.
+ */
+ rxq->rxq_consumer = rxq->rxq_first;
+ rxq->rxq_producer = rxq->rxq_first;
+ rxq->rxq_inuse = 0;
+ if (rxq->rxq_threshold < BCMETH_MINRXMBUFS)
+ rxq->rxq_threshold = BCMETH_MINRXMBUFS;
+
+ sc->sc_intmask |= RCVINT|RCVFIFOOF|RCVDESCUF;
+
+ /*
+ * Restart the receiver at the first descriptor
+ */
+ bcmeth_write_4(sc, rxq->rxq_reg_rcvaddrlo,
+ rxq->rxq_descmap->dm_segs[0].ds_addr);
+}
+
+static int
+bcmeth_rxq_attach(
+ struct bcmeth_softc *sc,
+ struct bcmeth_rxqueue *rxq,
+ u_int qno)
+{
+ size_t map_size = PAGE_SIZE;
+ size_t desc_count = map_size / sizeof(rxq->rxq_first[0]);
+ int error;
+ void *descs;
+
+ KASSERT(desc_count == 256 || desc_count == 512);
+
+ error = bcmeth_dmamem_alloc(sc->sc_dmat, map_size,
+ &rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs);
+ if (error)
+ return error;
+
+ memset(descs, 0, map_size);
+ rxq->rxq_first = descs;
+ rxq->rxq_last = rxq->rxq_first + desc_count;
+ rxq->rxq_consumer = descs;
+ rxq->rxq_producer = descs;
+
+ bcmeth_rxq_purge(sc, rxq, true);
+ bcmeth_rxq_reset(sc, rxq);
+
+ rxq->rxq_reg_rcvaddrlo = GMAC_RCVADDR_LOW;
+ rxq->rxq_reg_rcvctl = GMAC_RCVCONTROL;
+ rxq->rxq_reg_rcvptr = GMAC_RCVPTR;
+ rxq->rxq_reg_rcvsts0 = GMAC_RCVSTATUS0;
+
+ return 0;
+}
+
+static bool
+bcmeth_txq_active_p(
+ struct bcmeth_softc * const sc,
+ struct bcmeth_txqueue *txq)
+{
+ return !IF_IS_EMPTY(&txq->txq_mbufs);
+}
+
+static bool
+bcmeth_txq_fillable_p(
+ struct bcmeth_softc * const sc,
+ struct bcmeth_txqueue *txq)
+{
+ return txq->txq_free >= txq->txq_threshold;
}
static int
+bcmeth_txq_attach(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ u_int qno)
+{
+ size_t map_size = PAGE_SIZE;
+ size_t desc_count = map_size / sizeof(txq->txq_first[0]);
+ int error;
+ void *descs;
+
+ KASSERT(desc_count == 256 || desc_count == 512);
+
+ error = bcmeth_dmamem_alloc(sc->sc_dmat, map_size,
+ &txq->txq_descmap_seg, &txq->txq_descmap, &descs);
+ if (error)
+ return error;
+
+ memset(descs, 0, map_size);
+ txq->txq_first = descs;
+ txq->txq_last = txq->txq_first + desc_count;
+ txq->txq_consumer = descs;
+ txq->txq_producer = descs;
+
+ IFQ_SET_MAXLEN(&txq->txq_mbufs, BCMETH_MAXTXMBUFS);
+
+ txq->txq_reg_xmtaddrlo = GMAC_XMTADDR_LOW;
+ txq->txq_reg_xmtctl = GMAC_XMTCONTROL;
+ txq->txq_reg_xmtptr = GMAC_XMTPTR;
+ txq->txq_reg_xmtsts0 = GMAC_XMTSTATUS0;
+
+ bcmeth_txq_reset(sc, txq);
+
+ return 0;
+}
+
+static int
+bcmeth_txq_map_load(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ struct mbuf *m)
+{
+ bus_dmamap_t map;
+ int error;
+
+ map = M_GETCTX(m, bus_dmamap_t);
+ if (map != NULL)
+ return 0;
+
+ map = bcmeth_mapcache_get(sc, sc->sc_tx_mapcache);
+ if (map == NULL)
+ return ENOMEM;
+
+ error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
+ BUS_DMA_WRITE | BUS_DMA_NOWAIT);
+ if (error)
+ return error;
+
+ bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len,
+ BUS_DMASYNC_PREWRITE);
+ M_SETCTX(m, map);
+ return 0;
+}
+
+static void
+bcmeth_txq_map_unload(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ struct mbuf *m)
+{
+ KASSERT(m);
+ bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
+ bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(sc->sc_dmat, map);
+ bcmeth_mapcache_put(sc, sc->sc_tx_mapcache, map);
+}
+
+static bool
+bcmeth_txq_produce(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq,
+ struct mbuf *m)
+{
+ bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
+
+ if (map->dm_nsegs > txq->txq_free)
+ return false;
+
+ /*
+ * TCP Offload flag must be set in the first descriptor.
+ */
+ struct gmac_txdb *producer = txq->txq_producer;
+ uint32_t first_flags = TXDB_FLAG_SF;
+ uint32_t last_flags = TXDB_FLAG_EF;
+
+ /*
+ * If we've produced enough descriptors without consuming any
+ * we need to ask for an interrupt to reclaim some.
+ */
+ txq->txq_lastintr += map->dm_nsegs;
+ if (txq->txq_lastintr >= txq->txq_threshold
+ || txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) {
+ txq->txq_lastintr = 0;
+ last_flags |= TXDB_FLAG_IC;
+ }
+
+ KASSERT(producer != txq->txq_last);
+
+ struct gmac_txdb *start = producer;
+ size_t count = map->dm_nsegs;
+ producer->txdb_flags |= first_flags;
+ producer->txdb_addrlo = map->dm_segs[0].ds_addr;
+ producer->txdb_buflen = map->dm_segs[0].ds_len;
+ for (u_int i = 1; i < map->dm_nsegs; i++) {
+#if 0
+ printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first,
+ producer->txdb_flags, producer->txdb_buflen,
+ producer->txdb_addrlo, producer->txdb_addrhi);
+#endif
+ if (__predict_false(++producer == txq->txq_last)) {
+ bcmeth_txq_desc_presync(sc, txq, start,
+ txq->txq_last - start);
+ count -= txq->txq_last - start;
+ producer = txq->txq_first;
+ start = txq->txq_first;
+ }
+ producer->txdb_addrlo = map->dm_segs[i].ds_addr;
+ producer->txdb_buflen = map->dm_segs[i].ds_len;
+ }
+ producer->txdb_flags |= last_flags;
+#if 0
+ printf("[%zu]: %#x/%#x/%#x/%#x\n", producer - txq->txq_first,
+ producer->txdb_flags, producer->txdb_buflen,
+ producer->txdb_addrlo, producer->txdb_addrhi);
+#endif
+ bcmeth_txq_desc_presync(sc, txq, start, count);
+
+ /*
+ * Reduce free count by the number of segments we consumed.
+ */
+ txq->txq_free -= map->dm_nsegs;
+ KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer);
+ KASSERT(map->dm_nsegs == 1 || (txq->txq_producer->txdb_flags & TXDB_FLAG_EF) == 0);
+ KASSERT(producer->txdb_flags & TXDB_FLAG_EF);
+
+#if 0
+ printf("%s: mbuf %p: produced a %u byte packet in %u segments (%zd..%zd)\n",
+ __func__, m, m->m_pkthdr.len, map->dm_nsegs,
+ txq->txq_producer - txq->txq_first, producer - txq->txq_first);
+#endif
+
+ if (++producer == txq->txq_last)
+ txq->txq_producer = txq->txq_first;
+ else
+ txq->txq_producer = producer;
+ IF_ENQUEUE(&txq->txq_mbufs, m);
+ bpf_mtap(&sc->sc_if, m);
+
+ /*
+ * Let the transmitter know there's more to do
+ */
+ bcmeth_write_4(sc, txq->txq_reg_xmtptr,
+ txq->txq_descmap->dm_segs[0].ds_addr
+ + ((uintptr_t)txq->txq_producer & XMT_LASTDSCR));
+
+ return true;
+}
+
+static bool
+bcmeth_txq_enqueue(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq)
+{
+ for (;;) {
+ if (IF_QFULL(&txq->txq_mbufs))
+ return false;
+ struct mbuf *m = txq->txq_next;
+ if (m == NULL) {
+ int s = splnet();
+ IF_DEQUEUE(&sc->sc_if.if_snd, m);
+ splx(s);
+ if (m == NULL)
+ return true;
+ M_SETCTX(m, NULL);
+ } else {
+ txq->txq_next = NULL;
+ }
+ int error = bcmeth_txq_map_load(sc, txq, m);
+ if (error) {
+ aprint_error_dev(sc->sc_dev,
+ "discarded packet due to "
+ "dmamap load failure: %d\n", error);
+ m_freem(m);
+ continue;
+ }
+ KASSERT(txq->txq_next == NULL);
+ if (!bcmeth_txq_produce(sc, txq, m)) {
+ txq->txq_next = m;
+ return false;
+ }
+ KASSERT(txq->txq_next == NULL);
+ }
+}
+
+static bool
+bcmeth_txq_consume(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq)
+{
+ struct ifnet * const ifp = &sc->sc_if;
+ struct gmac_txdb *consumer = txq->txq_consumer;
+ size_t txfree = 0;
+
+#if 0
+ printf("%s: entry: free=%zu\n", __func__, txq->txq_free);
+#endif
+
+ for (;;) {
+ if (consumer == txq->txq_producer) {
+ txq->txq_consumer = consumer;
+ txq->txq_free += txfree;
+ txq->txq_lastintr -= min(txq->txq_lastintr, txfree);
+#if 0
+ printf("%s: empty: freed %zu descriptors going form %zu to %zu\n",
+ __func__, txfree, txq->txq_free - txfree, txq->txq_free);
+#endif
+ KASSERT(txq->txq_lastintr == 0);
+ KASSERT(txq->txq_free == txq->txq_last - txq->txq_first - 1);
+ return true;
+ }
+ bcmeth_txq_desc_postsync(sc, txq, consumer, 1);
+ uint32_t s0 = bcmeth_read_4(sc, txq->txq_reg_xmtsts0);
+ if (consumer == txq->txq_first + __SHIFTOUT(s0, XMT_CURRDSCR)) {
+ txq->txq_consumer = consumer;
+ txq->txq_free += txfree;
+ txq->txq_lastintr -= min(txq->txq_lastintr, txfree);
+#if 0
+ printf("%s: freed %zu descriptors\n",
+ __func__, txfree);
+#endif
+ return bcmeth_txq_fillable_p(sc, txq);
+ }
+
+ /*
+ * If this is the last descriptor in the chain, get the
+ * mbuf, free its dmamap, and free the mbuf chain itself.
+ */
+ const uint32_t txdb_flags = consumer->txdb_flags;
+ if (txdb_flags & TXDB_FLAG_EF) {
+ struct mbuf *m;
+
+ IF_DEQUEUE(&txq->txq_mbufs, m);
+ KASSERT(m);
+ bcmeth_txq_map_unload(sc, txq, m);
+#if 0
+ printf("%s: mbuf %p: consumed a %u byte packet\n",
+ __func__, m, m->m_pkthdr.len);
+#endif
+ ifp->if_opackets++;
+ ifp->if_obytes += m->m_pkthdr.len;
+ if (m->m_flags & M_MCAST)
+ ifp->if_omcasts++;
+ m_freem(m);
+ }
+
+ /*
+ * We own this packet again. Clear all flags except wrap.
+ */
+ txfree++;
+
+ /*
+ * Wrap at the last entry!
+ */
+ if (txdb_flags & TXDB_FLAG_ET) {
+ consumer->txdb_flags = TXDB_FLAG_ET;
+ KASSERT(consumer + 1 == txq->txq_last);
+ consumer = txq->txq_first;
+ } else {
+ consumer->txdb_flags = 0;
+ consumer++;
+ KASSERT(consumer < txq->txq_last);
+ }
+ }
+}
+
+static void
+bcmeth_txq_purge(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq)
+{
+ struct mbuf *m;
+ KASSERT((bcmeth_read_4(sc, UNIMAC_COMMAND_CONFIG) & TX_ENA) == 0);
+
+ for (;;) {
+ IF_DEQUEUE(&txq->txq_mbufs, m);
+ if (m == NULL)
+ break;
+ bcmeth_txq_map_unload(sc, txq, m);
+ m_freem(m);
+ }
+ if ((m = txq->txq_next) != NULL) {
+ txq->txq_next = NULL;
+ bcmeth_txq_map_unload(sc, txq, m);
+ m_freem(m);
+ }
+}
+
+static void
+bcmeth_txq_reset(
+ struct bcmeth_softc *sc,
+ struct bcmeth_txqueue *txq)
+{
+ /*
+ * sync all the descriptors
+ */
+ bcmeth_txq_desc_postsync(sc, txq, txq->txq_first,
+ txq->txq_last - txq->txq_first);
+
+ /*
+ * Make sure we own all descriptors in the ring.
+ */
+ struct gmac_txdb *txdb;
+ for (txdb = txq->txq_first; txdb < txq->txq_last - 1; txdb++) {
+ txdb->txdb_flags = 0;
+ }
+
+ /*
+ * Last descriptor has the wrap flag.
+ */
+ txdb->txdb_flags = TXDB_FLAG_ET;
+
+ /*
+ * Reset the producer consumer indexes.
+ */
+ txq->txq_consumer = txq->txq_first;
+ txq->txq_producer = txq->txq_first;
+ txq->txq_free = txq->txq_last - txq->txq_first - 1;
+ txq->txq_threshold = txq->txq_free / 2;
+ txq->txq_lastintr = 0;
+
+ /*
+ * What do we want to get interrupted on?
+ */
+ sc->sc_intmask |= XMTINT_0 | XMTUF;
+
+ /*
+ * Restart the transmiter at the first descriptor
+ */
+ bcmeth_write_4(sc, txq->txq_reg_xmtaddrlo,
+ txq->txq_descmap->dm_segs->ds_addr);
+}
+
+static void
+bcmeth_ifstart(struct ifnet *ifp)
+{
+ struct bcmeth_softc * const sc = ifp->if_softc;
+
+ atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR);
+ softint_schedule(sc->sc_soft_ih);
+}
+
+int
bcmeth_intr(void *arg)
{
struct bcmeth_softc * const sc = arg;
+ uint32_t soft_flags = 0;
int rv = 0;
mutex_enter(sc->sc_hwlock);
- uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS);
- bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus);
+ sc->sc_ev_intr.ev_count++;
+
+ for (;;) {
+ uint32_t intstatus = bcmeth_read_4(sc, GMAC_INTSTATUS);
+ intstatus &= sc->sc_intmask;
+ bcmeth_write_4(sc, GMAC_INTSTATUS, intstatus); /* write 1 to clear */
+ if (intstatus == 0) {
+ break;
+ }
+#if 0
+ aprint_normal_dev(sc->sc_dev, "%s: ievent=%#x intmask=%#x\n",
+ __func__, ievent, bcmeth_read_4(sc, GMAC_INTMASK));
+#endif
+ if (intstatus & RCVINT) {
+ intstatus &= ~RCVINT;
+ sc->sc_intmask &= ~RCVINT;
+ soft_flags |= SOFT_RXINTR;
+ }
+
+ if (intstatus & XMTINT_0) {
+ intstatus &= ~XMTINT_0;
+ sc->sc_intmask &= ~XMTINT_0;
+ soft_flags |= SOFT_TXINTR;
+ }
+
+ if (intstatus & RCVDESCUF) {
+ intstatus &= ~RCVDESCUF;
+ sc->sc_intmask &= ~RCVDESCUF;
+ soft_flags |= SOFT_RXUNDERFLOW;
+ }
+
+ if (intstatus) {
+ aprint_error_dev(sc->sc_dev, "intr: intstatus=%#x\n",
+ intstatus);
+ Debugger();
+ sc->sc_intmask &= ~intstatus;
+ soft_flags |= SOFT_REINIT;
+ break;
+ }
+ }
+
+ if (soft_flags) {
+ bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask);
+ atomic_or_uint(&sc->sc_soft_flags, soft_flags);
+ softint_schedule(sc->sc_soft_ih);
+ rv = 1;
+ }
mutex_exit(sc->sc_hwlock);
return rv;
}
+
+void
+bcmeth_soft_intr(void *arg)
+{
+ struct bcmeth_softc * const sc = arg;
+ struct ifnet * const ifp = &sc->sc_if;
+
+ mutex_enter(sc->sc_lock);
+
+ u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0);
+
+ sc->sc_ev_soft_intr.ev_count++;
+
+ if (soft_flags & SOFT_REINIT) {
+ int s = splnet();
+ bcmeth_ifinit(ifp);
+ splx(s);
+ soft_flags = 0;
+ }
+
+ if (soft_flags & SOFT_RXUNDERFLOW) {
+ struct bcmeth_rxqueue * const rxq = &sc->sc_rxq;
+ size_t threshold = 5 * rxq->rxq_threshold / 4;
+ if (threshold >= rxq->rxq_last - rxq->rxq_first) {
+ threshold = rxq->rxq_last - rxq->rxq_first - 1;
+ } else {
+ sc->sc_intmask |= RCVDESCUF;
+ }
+ aprint_normal_dev(sc->sc_dev,
+ "increasing receive buffers from %zu to %zu\n",
+ rxq->rxq_threshold, threshold);
+ rxq->rxq_threshold = threshold;
+ }
+
+ if ((soft_flags & SOFT_TXINTR)
+ || bcmeth_txq_active_p(sc, &sc->sc_txq)) {
+ /*
+ * Let's do what we came here for. Consume transmitted
+ * packets off the the transmit ring.
+ */
+ if (!bcmeth_txq_consume(sc, &sc->sc_txq)
+ || !bcmeth_txq_enqueue(sc, &sc->sc_txq)) {
+ sc->sc_ev_tx_stall.ev_count++;
+ ifp->if_flags |= IFF_OACTIVE;
+ } else {
+ ifp->if_flags &= ~IFF_OACTIVE;
+ }
+ sc->sc_intmask |= XMTINT_0;
+ }
+
+ if (soft_flags & (SOFT_RXINTR|SOFT_RXUNDERFLOW)) {
+ /*
+ * Let's consume
+ */
+ bcmeth_rxq_consume(sc, &sc->sc_rxq);
+ sc->sc_intmask |= RCVINT;
+ }
+
+ if (ifp->if_flags & IFF_RUNNING) {
+ bcmeth_rxq_produce(sc, &sc->sc_rxq);
+ bcmeth_write_4(sc, GMAC_INTMASK, sc->sc_intmask);
+ } else {
+ KASSERT((soft_flags & SOFT_RXUNDERFLOW) == 0);
+ }
+
+ mutex_exit(sc->sc_lock);
+}
