> Date: Tue, 10 Oct 2023 19:40:31 +0200
> From: Stefan Sperling <[email protected]>
>
> This patch adds enough code to get Elkart Lake devices with PCI
> Vendor ID 8086 and Product ID 4ba0 to attach and pass traffic.
>
> dwqe0 at pci0 dev 29 function 1 "Intel Elkhart Lake Ethernet" rev 0x11: rev
> 0x52, address xx:xx:xx:xx:xx:xx
> eephy0 at dwqe0 phy 1: 88E1512 10/100/1000 PHY, rev. 1
>
> # ifconfig dwqe0 media
> dwqe0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> mtu 1500
> lladdr xx:xx:xx:xx:xx:xx
> index 5 priority 0 llprio 3
> media: Ethernet autoselect (1000baseT full-duplex,master)
> status: active
> supported media:
> media none
> media 10baseT
> media 10baseT mediaopt full-duplex
> media 100baseTX
> media 100baseTX mediaopt full-duplex
> media 1000baseT
> media 1000baseT mediaopt full-duplex
> media autoselect
>
> There is one issue though: Tx speed is currently capped to 64 Mbps in my
> testing, while Rx works at 300 Mbps. I will keep investigating this problem.
> But since 64 is better than 0 and the device seems to be working reliably
> in my testing I think it's worth getting this in and fixing remaining the
> issues in the tree.
>
> Various other PCI IDs could attach to this but would require more code.
> I don't have access to such hardware. So they remain commented for now.
> There is a second Elkhart Lake Ethernet device in the same box, with
> PCI id 4bb0. However, this interface has no physical Ethernet port
> exposed, and according to Linux there is no PHY attached to the MAC.
> Linux detects this device but fails to atach a PHY.
>
> ok?
So the GMAC_VERSION #define is simply wrong. We should commit the
diff attached and drop the sc_core stuff you added below.
A few more nits below...
> diff refs/heads/master refs/heads/dwqe
> commit - ef065493fb463fe122ab7b535932b871830b6de3
> commit + a82b4a31a1787b55291d125b3adfb40fe58779e4
> blob - c6094ca5a57d964784f56124b8d923393b1abb66
> blob + cf64e8ff2018fb0ee1e7e59b29b2bc2dd281aa99
> --- sys/arch/amd64/conf/GENERIC
> +++ sys/arch/amd64/conf/GENERIC
> @@ -556,6 +556,7 @@ lii* at pci? # Attansic L2
> Ethernet
> jme* at pci? # JMicron JMC250/JMC260 Ethernet
> bnxt* at pci? # Broadcom BCM573xx, BCM574xx
> ixl* at pci? # Intel Ethernet 700 Series
> +dwqe* at pci? # Intel Elkhart Lake Ethernet
> mcx* at pci? # Mellanox ConnectX-4
> iavf* at pci? # Intel Ethernet Adaptive VF
> aq* at pci? # Aquantia aQtion Ethernet
> blob - 7e6afaba34dd35e3bd180055a68741ee3b773486
> blob + 6a4cc88f048d63430ea9eb6080cd98741f7e1612
> --- sys/dev/fdt/if_dwqe_fdt.c
> +++ sys/dev/fdt/if_dwqe_fdt.c
> @@ -111,6 +111,8 @@ dwqe_fdt_attach(struct device *parent, struct device *
> }
> sc->sc_dmat = faa->fa_dmat;
>
> + sc->sc_core = DWQE_CORE_GMAC;
> +
> /* Decide GMAC id through address */
> switch (faa->fa_reg[0].addr) {
> case 0xfe2a0000: /* RK3568 */
> blob - 7d260ef46054d6566ef2f81f6cf96dc8de5a5893
> blob + 7ae36ee6e804b7456c296a248d2b058ffe65f722
> --- sys/dev/ic/dwqe.c
> +++ sys/dev/ic/dwqe.c
> @@ -100,7 +100,14 @@ dwqe_attach(struct dwqe_softc *sc)
> uint32_t version, mode;
> int i;
>
> - version = dwqe_read(sc, GMAC_VERSION);
> + switch (sc->sc_core) {
> + case DWQE_CORE_GMAC:
> + version = dwqe_read(sc, GMAC_VERSION);
> + break;
> + case DWQE_CORE_GMAC4:
> + version = dwqe_read(sc, GMAC4_VERSION);
> + break;
> + }
> printf(": rev 0x%02x, address %s\n", version & GMAC_VERSION_SNPS_MASK,
> ether_sprintf(sc->sc_lladdr));
>
> @@ -705,7 +712,7 @@ dwqe_up(struct dwqe_softc *sc)
> {
> struct ifnet *ifp = &sc->sc_ac.ac_if;
> struct dwqe_buf *txb, *rxb;
> - uint32_t mode, reg, tqs, rqs;
> + uint32_t mode, reg, fifosz, tqs, rqs;
> int i;
>
> /* Allocate Tx descriptor ring. */
> @@ -793,9 +800,21 @@ dwqe_up(struct dwqe_softc *sc)
> mode |= GMAC_MTL_CHAN_RX_OP_MODE_RSF;
> }
> mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RQS_MASK;
> - rqs = (128 << GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(sc->sc_hw_feature[1]) /
> - 256) - 1;
> - mode |= rqs << GMAC_MTL_CHAN_RX_OP_MODE_RQS_SHIFT;
> + if (sc->sc_rxfifo_size)
> + fifosz = sc->sc_rxfifo_size;
> + else
> + fifosz = (128 <<
> + GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(sc->sc_hw_feature[1]));
> + rqs = fifosz / 256 - 1;
> + mode |= (rqs << GMAC_MTL_CHAN_RX_OP_MODE_RQS_SHIFT) &
> + GMAC_MTL_CHAN_RX_OP_MODE_RQS_MASK;
> + if (fifosz >= 4096) {
> + mode |= GMAC_MTL_CHAN_RX_OP_MODE_EHFC;
> + mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RFD_MASK;
> + mode |= 0x3 << GMAC_MTL_CHAN_RX_OP_MODE_RFD_SHIFT;
> + mode &= ~GMAC_MTL_CHAN_RX_OP_MODE_RFA_MASK;
> + mode |= 0x1 << GMAC_MTL_CHAN_RX_OP_MODE_RFA_SHIFT;
> + }
> dwqe_write(sc, GMAC_MTL_CHAN_RX_OP_MODE(0), mode);
>
> mode = dwqe_read(sc, GMAC_MTL_CHAN_TX_OP_MODE(0));
> @@ -809,9 +828,14 @@ dwqe_up(struct dwqe_softc *sc)
> mode &= ~GMAC_MTL_CHAN_TX_OP_MODE_TXQEN_MASK;
> mode |= GMAC_MTL_CHAN_TX_OP_MODE_TXQEN;
> mode &= ~GMAC_MTL_CHAN_TX_OP_MODE_TQS_MASK;
> - tqs = (128 << GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(sc->sc_hw_feature[1]) /
> - 256) - 1;
> - mode |= tqs << GMAC_MTL_CHAN_TX_OP_MODE_TQS_SHIFT;
> + if (sc->sc_txfifo_size)
> + fifosz = sc->sc_txfifo_size;
> + else
> + fifosz = (128 <<
> + GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(sc->sc_hw_feature[1]));
> + tqs = (fifosz / 256) - 1;
> + mode |= (tqs << GMAC_MTL_CHAN_TX_OP_MODE_TQS_SHIFT) &
> + GMAC_MTL_CHAN_TX_OP_MODE_TQS_MASK;
> dwqe_write(sc, GMAC_MTL_CHAN_TX_OP_MODE(0), mode);
>
> reg = dwqe_read(sc, GMAC_QX_TX_FLOW_CTRL(0));
> blob - d35a811571a16d98ca672d17ec188111d4d92132
> blob + d43c9d12b27b2f871fb7105f99a1fe99a32ec9d9
> --- sys/dev/ic/dwqereg.h
> +++ sys/dev/ic/dwqereg.h
> @@ -41,6 +41,7 @@
> #define GMAC_MAC_HASH_TAB_REG1 0x0014
> #define GMAC_VERSION 0x0020
> #define GMAC_VERSION_SNPS_MASK 0xff
> +#define GMAC4_VERSION 0x0110
> #define GMAC_INT_MASK 0x003c
> #define GMAC_INT_MASK_LPIIM (1 << 10)
> #define GMAC_INT_MASK_PIM (1 << 3)
> blob - 68d698a50beef00f002d986d7d24f8984c3efe0c
> blob + 8d2c534793f9bfa5f2895ef5a44ec8e927d4fc00
> --- sys/dev/ic/dwqevar.h
> +++ sys/dev/ic/dwqevar.h
> @@ -24,6 +24,11 @@ enum dwqe_phy_mode {
> DWQE_PHY_MODE_RGMII_RXID,
> };
>
> +enum dwqe_core {
> + DWQE_CORE_GMAC,
> + DWQE_CORE_GMAC4
> +};
> +
> struct dwqe_buf {
> bus_dmamap_t tb_map;
> struct mbuf *tb_m;
> @@ -60,6 +65,7 @@ struct dwqe_softc {
> int sc_link;
> int sc_phyloc;
> enum dwqe_phy_mode sc_phy_mode;
> + enum dwqe_core sc_core;
> struct timeout sc_phy_tick;
> int sc_fixed_link;
>
> @@ -97,6 +103,8 @@ struct dwqe_softc {
> int sc_pbl;
> int sc_txpbl;
> int sc_rxpbl;
> + int sc_txfifo_size;
> + int sc_rxfifo_size;
> int sc_axi_config;
> int sc_lpi_en;
> int sc_xit_frm;
> blob - 101ed502e76987c27878712b4921cc49f7eb7f59
> blob + 6868eb3591995804dbc332308c9d9629f73c269f
> --- sys/dev/pci/files.pci
> +++ sys/dev/pci/files.pci
> @@ -363,6 +363,10 @@ device ixl: ether, ifnet, ifmedia, intrmap, stoeplitz
> attach ixl at pci
> file dev/pci/if_ixl.c ixl
>
> +# Intel Elkhart Lake Ethernet
> +attach dwqe at pci with dwqe_pci
> +file dev/pci/if_dwqe_pci.c dwqe_pci
> +
> # Neterion Xframe 10 Gigabit ethernet
> device xge: ether, ifnet, ifmedia
> attach xge at pci
> blob - /dev/null
> blob + 101b5670451611858f70b95a1082442b47056600 (mode 644)
> --- /dev/null
> +++ sys/dev/pci/if_dwqe_pci.c
> @@ -0,0 +1,155 @@
> +/* $OpenBSD$ */
> +
> +/*
> + * Copyright (c) 2023 Stefan Sperling <[email protected]>
> + *
> + * Permission to use, copy, modify, and distribute this software for any
> + * purpose with or without fee is hereby granted, provided that the above
> + * copyright notice and this permission notice appear in all copies.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> + */
> +
> +/*
> + * Driver for the Intel Elkhart Lake ethernet controller.
> + */
> +
> +#include <sys/param.h>
> +#include <sys/conf.h>
> +#include <sys/systm.h>
> +#include <sys/device.h>
> +#include <sys/kernel.h>
> +#include <sys/malloc.h>
> +#include <sys/mbuf.h>
> +#include <sys/queue.h>
> +#include <sys/socket.h>
> +#include <sys/sockio.h>
> +#include <sys/timeout.h>
> +#include <sys/task.h>
> +
> +#include <dev/pci/pcireg.h>
> +#include <dev/pci/pcivar.h>
> +#include <dev/pci/pcidevs.h>
> +
> +#if NBPFILTER > 0
> +#include <net/bpf.h>
> +#endif
> +#include <net/if.h>
> +#include <net/if_dl.h>
> +#include <net/if_media.h>
> +
> +#include <netinet/in.h>
> +#include <netinet/if_ether.h>
> +
> +#include <dev/mii/mii.h>
> +#include <dev/mii/miivar.h>
> +
> +#include <dev/ic/dwqereg.h>
> +#include <dev/ic/dwqevar.h>
> +
> +static const struct pci_matchid dwqe_pci_devices[] = {
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_RGMII_1G },
> +#if 0
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_SGMII_1G },
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE0_SGMII_2G },
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_RGMII_1G },
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_SGMII_1G },
> + { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_EHL_PSE1_SGMII_2G },
> +#endif
> +};
> +
> +struct dwqe_pci_softc {
> + struct dwqe_softc sc_sc;
> + pci_chipset_tag_t sc_pct;
> + pcitag_t sc_pcitag;
> + bus_size_t sc_mapsize;
> +};
> +
> +int
> +dwqe_pci_match(struct device *parent, void *cfdata, void *aux)
> +{
> + struct pci_attach_args *pa = aux;
> + return pci_matchbyid(pa, dwqe_pci_devices, nitems(dwqe_pci_devices));
> +}
> +
> +void
> +dwqe_pci_attach(struct device *parent, struct device *self, void *aux)
> +{
> + struct pci_attach_args *pa = aux;
> + struct dwqe_pci_softc *psc = (void *)self;
> + struct dwqe_softc *sc = &psc->sc_sc;
> + pci_intr_handle_t ih;
> + pcireg_t memtype;
> + int err;
> + const char *intrstr;
> +
> + psc->sc_pct = pa->pa_pc;
> + psc->sc_pcitag = pa->pa_tag;
> +
> + sc->sc_dmat = pa->pa_dmat;
> +
> + memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, PCI_MAPREG_START);
> + err = pci_mapreg_map(pa, PCI_MAPREG_START, memtype, 0,
> + &sc->sc_iot, &sc->sc_ioh, NULL, &psc->sc_mapsize, 0);
> + if (err) {
> + printf("%s: can't map mem space\n", DEVNAME(sc));
> + return;
> + }
> +
> + if (pci_intr_map_msi(pa, &ih) && pci_intr_map(pa, &ih)) {
> + printf("%s: can't map interrupt\n", DEVNAME(sc));
> + return;
> + }
> +
> + intrstr = pci_intr_string(psc->sc_pct, ih);
> + sc->sc_ih = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE,
> + dwqe_intr, psc, sc->sc_dev.dv_xname);
> + if (sc->sc_ih == NULL) {
> + printf(": can't establish interrupt");
> + if (intrstr != NULL)
> + printf(" at %s", intrstr);
> + printf("\n");
> + return;
> + }
> +
> + switch (PCI_PRODUCT(pa->pa_id)) {
> + case PCI_PRODUCT_INTEL_EHL_PSE0_RGMII_1G:
> + sc->sc_phy_mode = DWQE_PHY_MODE_RGMII;
This suggests that the PHY must be programmed to disable its internal
delays. Our eephy(4) driver doesn't do this. That may explain the
poor performance. See how MIIF_SETDELAY, MIIF_RXID and MIIF_TXID are
used in rgephy(4).
Anyway, that is a different driver, so a separate diff that can go in
after this one.
> + sc->sc_clk = GMAC_MAC_MDIO_ADDR_CR_250_300;
> + sc->sc_clkrate = 200000000;
> + break;
> + default:
> + sc->sc_phy_mode = DWQE_PHY_MODE_UNKNOWN;
> + break;
> + }
> +
> + sc->sc_phyloc = MII_PHY_ANY;
> + sc->sc_core = DWQE_CORE_GMAC4;
> + sc->sc_8xpbl = 1;
> + sc->sc_txpbl = 32;
> + sc->sc_rxpbl = 32;
> + sc->sc_txfifo_size = 4096 * 8;
> + sc->sc_rxfifo_size = 4096 * 8;
Makes more sense to simply use 32768 here. Intel's data sheet says 32KB.
> +
> + sc->sc_axi_config = 1;
> + sc->sc_wr_osr_lmt = 1;
> + sc->sc_rd_osr_lmt = 1;
> + sc->sc_blen[0] = 4;
> + sc->sc_blen[1] = 8;
> + sc->sc_blen[2] = 16;
> +
> + dwqe_lladdr_read(sc, sc->sc_lladdr);
> +
> + dwqe_reset(sc);
> + dwqe_attach(sc);
> +}
> +
> +const struct cfattach dwqe_pci_ca = {
> + sizeof(struct dwqe_softc), dwqe_pci_match, dwqe_pci_attach
> +};
>
>
Index: dev/ic/dwqereg.h
===================================================================
RCS file: /cvs/src/sys/dev/ic/dwqereg.h,v
retrieving revision 1.2
diff -u -p -r1.2 dwqereg.h
--- dev/ic/dwqereg.h 16 Feb 2023 14:43:53 -0000 1.2
+++ dev/ic/dwqereg.h 10 Oct 2023 18:40:19 -0000
@@ -39,8 +39,6 @@
#define GMAC_MAC_PACKET_FILTER_PR (1 << 0)
#define GMAC_MAC_HASH_TAB_REG0 0x0010
#define GMAC_MAC_HASH_TAB_REG1 0x0014
-#define GMAC_VERSION 0x0020
-#define GMAC_VERSION_SNPS_MASK 0xff
#define GMAC_INT_MASK 0x003c
#define GMAC_INT_MASK_LPIIM (1 << 10)
#define GMAC_INT_MASK_PIM (1 << 3)
@@ -60,6 +58,8 @@
#define GMAC_INT_STATUS 0x00b0
#define GMAC_INT_EN 0x00b4
#define GMAC_MAC_1US_TIC_CTR 0x00dc
+#define GMAC_VERSION 0x0110
+#define GMAC_VERSION_SNPS_MASK 0xff
#define GMAC_MAC_HW_FEATURE(x) (0x011c + (x) * 0x4)
#define GMAC_MAC_HW_FEATURE1_TXFIFOSIZE(x) (((x) >> 6) & 0x1f)
#define GMAC_MAC_HW_FEATURE1_RXFIFOSIZE(x) (((x) >> 0) & 0x3f)
