The branch main has been updated by dab: URL: https://cgit.FreeBSD.org/src/commit/?id=70450ecd37fa4ce06bd957195b00669dc3445e04
commit 70450ecd37fa4ce06bd957195b00669dc3445e04 Author: Austin Zhang <[email protected]> AuthorDate: 2024-02-07 18:55:02 +0000 Commit: David Bright <[email protected]> CommitDate: 2024-02-07 21:14:58 +0000 ntb: Add Intel Xeon Gen4 support The NTB hardware of XEON Ice lake and Sapphire Rapids has register mapping changes Add a new NTB_XEON_GEN4 device type and use it to conditionalize driver logic differs Reviewed by: vangyzen, dab Sponsored by: Dell Technologies Differential Revision: https://reviews.freebsd.org/D43291 --- sys/dev/ntb/ntb_hw/ntb_hw_intel.c | 462 +++++++++++++++++++++++++++++++++++--- sys/dev/ntb/ntb_hw/ntb_hw_intel.h | 115 ++++++++++ 2 files changed, 543 insertions(+), 34 deletions(-) diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c index 5c07fb8d7e7d..d4852917085d 100644 --- a/sys/dev/ntb/ntb_hw/ntb_hw_intel.c +++ b/sys/dev/ntb/ntb_hw/ntb_hw_intel.c @@ -87,6 +87,7 @@ enum ntb_device_type { NTB_XEON_GEN1, NTB_XEON_GEN3, + NTB_XEON_GEN4, NTB_ATOM }; @@ -171,6 +172,7 @@ struct ntb_reg { struct ntb_alt_reg { uint32_t db_bell; uint32_t db_mask; + uint32_t db_clear; uint32_t spad; }; @@ -356,9 +358,12 @@ static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); static void intel_ntb_detect_max_mw(struct ntb_softc *ntb); static int intel_ntb_detect_xeon(struct ntb_softc *ntb); static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb); +static int intel_ntb_detect_xeon_gen4(struct ntb_softc *ntb); +static int intel_ntb_detect_xeon_gen4_cfg(struct ntb_softc *ntb); static int intel_ntb_detect_atom(struct ntb_softc *ntb); static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb); +static int intel_ntb_xeon_gen4_init_dev(struct ntb_softc *ntb); static int intel_ntb_atom_init_dev(struct ntb_softc *ntb); static void intel_ntb_teardown_xeon(struct ntb_softc *ntb); static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); @@ -371,6 +376,7 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, static int xeon_setup_b2b_mw(struct ntb_softc *, const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); static int xeon_gen3_setup_b2b_mw(struct ntb_softc *); +static int xeon_gen4_setup_b2b_mw(struct ntb_softc *); static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size); static inline bool link_is_up(struct ntb_softc *ntb); @@ -482,12 +488,15 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, #define NTB_SDOORBELL_LOCKUP (1 << 1) #define NTB_SB01BASE_LOCKUP (1 << 2) #define NTB_B2BDOORBELL_BIT14 (1 << 3) +#define NTB_BAR_ALIGN (1 << 4) +#define NTB_LTR_BAD (1 << 5) /* Software/configuration owns the top 16 bits. */ #define NTB_SPLIT_BAR (1ull << 16) #define NTB_ONE_MSIX (1ull << 17) #define NTB_FEATURES_STR \ - "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ + "\20\21SPLIT_BAR4\06LTR_BAD\05BAR_ALIGN" \ + "\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" static struct ntb_hw_info pci_ids[] = { @@ -511,6 +520,9 @@ static struct ntb_hw_info pci_ids[] = { { 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B", NTB_XEON_GEN3, 0 }, + + { 0x347e8086, "ICX/SPR Xeon Non-Transparent Bridge B2B", + NTB_XEON_GEN4, 0 }, }; static const struct ntb_reg atom_reg = { @@ -626,6 +638,32 @@ static const struct ntb_xlat_reg xeon_gen3_sec_xlat = { .bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE, }; +static const struct ntb_reg xeon_gen4_reg = { + .ntb_ctl = XEON_GEN4_REG_IMNTB_CTL, + .lnk_sta = XEON_GEN4_REG_LINK_STATUS, /* mmio */ + .db_size = sizeof(uint32_t), + .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, +}; + +static const struct ntb_alt_reg xeon_gen4_pri_reg = { + .db_clear = XEON_GEN4_REG_IMINT_STATUS, + .db_mask = XEON_GEN4_REG_IMINT_DISABLE, + .spad = XEON_GEN4_REG_IMSPAD, +}; + +static const struct ntb_alt_reg xeon_gen4_b2b_reg = { + .db_bell = XEON_GEN4_REG_IMDOORBELL, + .spad = XEON_GEN4_REG_EMSPAD, +}; + +static const struct ntb_xlat_reg xeon_gen4_sec_xlat = { + .bar2_limit = XEON_GEN4_REG_IMBAR1XLIMIT, + .bar2_xlat = XEON_GEN4_REG_IMBAR1XBASE, + + .bar4_limit = XEON_GEN4_REG_IMBAR1XLIMIT, + .bar4_xlat = XEON_GEN4_REG_IMBAR2XBASE, +}; + SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "B2B MW segment overrides -- MUST be the same on both sides"); @@ -704,6 +742,8 @@ intel_ntb_attach(device_t device) error = intel_ntb_detect_atom(ntb); else if (ntb->type == NTB_XEON_GEN3) error = intel_ntb_detect_xeon_gen3(ntb); + else if (ntb->type == NTB_XEON_GEN4) + error = intel_ntb_detect_xeon_gen4(ntb); else error = intel_ntb_detect_xeon(ntb); if (error != 0) @@ -720,6 +760,8 @@ intel_ntb_attach(device_t device) error = intel_ntb_atom_init_dev(ntb); else if (ntb->type == NTB_XEON_GEN3) error = intel_ntb_xeon_gen3_init_dev(ntb); + else if (ntb->type == NTB_XEON_GEN4) + error = intel_ntb_xeon_gen4_init_dev(ntb); else error = intel_ntb_xeon_init_dev(ntb); if (error != 0) @@ -873,6 +915,10 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb) bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ; bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ; bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE; + } else if (ntb->type == NTB_XEON_GEN4) { + bar->psz_off = XEON_GEN4_CFG_REG_IMBAR1SZ; + bar->ssz_off = XEON_GEN4_CFG_REG_EMBAR1SZ; + bar->pbarxlat_off = XEON_GEN4_REG_EXT_BAR1BASE; } else { bar->psz_off = XEON_PBAR23SZ_OFFSET; bar->ssz_off = XEON_SBAR23SZ_OFFSET; @@ -888,6 +934,10 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb) bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ; bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ; bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE; + } else if (ntb->type == NTB_XEON_GEN4) { + bar->psz_off = XEON_GEN4_CFG_REG_IMBAR2SZ; + bar->ssz_off = XEON_GEN4_CFG_REG_EMBAR2SZ; + bar->pbarxlat_off = XEON_GEN4_REG_EXT_BAR2BASE; } else { bar->psz_off = XEON_PBAR4SZ_OFFSET; bar->ssz_off = XEON_SBAR4SZ_OFFSET; @@ -897,7 +947,8 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb) if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) goto out; - if (ntb->type == NTB_XEON_GEN3) { + if (ntb->type == NTB_XEON_GEN3 || + ntb->type == NTB_XEON_GEN4) { device_printf(ntb->device, "no split bar support\n"); return (ENXIO); } @@ -1176,7 +1227,63 @@ intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb) return (ENXIO); } - return (0); + return (rc); +} + +static int +intel_ntb_xeon_gen4_init_isr(struct ntb_softc *ntb) +{ + uint64_t i, reg; + uint32_t desired_vectors, num_vectors; + int rc; + + ntb->allocated_interrupts = 0; + ntb->last_ts = ticks; + + /* Mask all the interrupts, including hardware interrupt */ + intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_DISABLE, ~0ULL); + + /* Clear Interrupt Status */ + reg = intel_ntb_reg_read(8, XEON_GEN4_REG_IMINT_STATUS); + intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_STATUS, reg); + + num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), + XEON_GEN4_DB_MSIX_VECTOR_COUNT); + + rc = pci_alloc_msix(ntb->device, &num_vectors); + if (rc != 0) { + device_printf(ntb->device, + "Interrupt allocation failed %d\n", rc); + return (rc); + } + if (desired_vectors != num_vectors) { + device_printf(ntb->device, "Couldn't get %d vectors\n", + XEON_GEN4_DB_MSIX_VECTOR_COUNT); + return (ENXIO); + } + if (num_vectors != XEON_GEN4_DB_MSIX_VECTOR_COUNT) { + device_printf(ntb->device, + "Need to remap interrupts, giving up\n"); + return (ENXIO); + } + + /* + * The MSIX vectors and the interrupt status bits are not lined up + * on Gen3 (Skylake) and Gen4. By default the link status bit is bit + * 32, however it is by default MSIX vector0. We need to fixup to + * line them up. The vectors at reset is 1-32,0. We need to reprogram + * to 0-32. + */ + for (i = 0; i < XEON_GEN4_DB_MSIX_VECTOR_COUNT; i++) + intel_ntb_reg_write(1, XEON_GEN4_REG_INTVEC + i, i); + + intel_ntb_create_msix_vec(ntb, num_vectors); + rc = intel_ntb_setup_msix(ntb, num_vectors); + + /* enable all interrupts */ + intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_DISABLE, 0ULL); + + return (rc); } static int @@ -1304,6 +1411,7 @@ db_ioread(struct ntb_softc *ntb, uint64_t regoff) switch (ntb->type) { case NTB_ATOM: case NTB_XEON_GEN3: + case NTB_XEON_GEN4: return (intel_ntb_reg_read(8, regoff)); case NTB_XEON_GEN1: return (intel_ntb_reg_read(2, regoff)); @@ -1332,6 +1440,7 @@ db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) switch (ntb->type) { case NTB_ATOM: case NTB_XEON_GEN3: + case NTB_XEON_GEN4: intel_ntb_reg_write(8, regoff, val); break; case NTB_XEON_GEN1: @@ -1448,7 +1557,8 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) ntb->last_ts = ticks; vec_mask = intel_ntb_vec_mask(ntb, vec); - if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX) + if ((ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) && + vec == XEON_GEN3_LINK_VECTOR_INDEX) vec_mask |= ntb->db_link_mask; if ((vec_mask & ntb->db_link_mask) != 0) { if (intel_ntb_poll_link(ntb)) @@ -1456,6 +1566,9 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) if (ntb->type == NTB_XEON_GEN3) intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS)); + if (ntb->type == NTB_XEON_GEN4) + intel_ntb_reg_write(8, XEON_GEN4_REG_IMINT_STATUS, + intel_ntb_reg_read(8, XEON_GEN4_REG_IMINT_STATUS)); } if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && @@ -1591,6 +1704,7 @@ intel_ntb_detect_max_mw(struct ntb_softc *ntb) ntb->mw_count = XEON_SNB_MW_COUNT; break; case NTB_XEON_GEN3: + case NTB_XEON_GEN4: if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT; else @@ -1720,6 +1834,86 @@ intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb) return (0); } +static int +intel_ntb_is_ICX(struct ntb_softc *ntb) +{ + uint8_t revision; + + revision = pci_get_revid(ntb->device); + if (ntb->type == NTB_XEON_GEN4 && + revision >= PCI_DEV_REV_ICX_MIN && + revision <= PCI_DEV_REV_ICX_MAX) + return (1); + + return (0); +} + +static int +intel_ntb_is_SPR(struct ntb_softc *ntb) +{ + uint8_t revision; + + revision = pci_get_revid(ntb->device); + if (ntb->type == NTB_XEON_GEN4 && + revision > PCI_DEV_REV_ICX_MAX) + return (1); + + return (0); +} + +static int +intel_ntb_detect_xeon_gen4(struct ntb_softc *ntb) +{ + if (intel_ntb_is_ICX(ntb)) { + ntb->features |= NTB_BAR_ALIGN; + ntb->features |= NTB_LTR_BAD; + } + return (0); +} + +static int +intel_ntb_detect_xeon_gen4_cfg(struct ntb_softc *ntb) +{ + uint32_t ppd1; + + ppd1 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD1); + ntb->ppd = ppd1; + if (intel_ntb_is_ICX(ntb)) { + if ((ppd1 & GEN4_PPD_TOPO_MASK) == GEN4_PPD_TOPO_B2B_USD) { + /* NTB Port is configured as USD/DSP */ + ntb->conn_type = NTB_CONN_B2B; + ntb->dev_type = NTB_DEV_USD; + } else if ((ppd1 & GEN4_PPD_TOPO_MASK) == GEN4_PPD_TOPO_B2B_DSD) { + /* NTB Port is configured as DSD/USP */ + ntb->conn_type = NTB_CONN_B2B; + ntb->dev_type = NTB_DEV_DSD; + } else { + device_printf(ntb->device, "Unsupported connection type: %u\n", + (ppd1 & GEN4_PPD_CONN_MASK)); + return (ENXIO); + } + } else if (intel_ntb_is_SPR(ntb)) { + if ((ppd1 & SPR_PPD_TOPO_MASK) == SPR_PPD_TOPO_B2B_USD) { + /* NTB Port is configured as USD/DSP */ + ntb->conn_type = NTB_CONN_B2B; + ntb->dev_type = NTB_DEV_USD; + } else if ((ppd1 & SPR_PPD_TOPO_MASK) == SPR_PPD_TOPO_B2B_DSD) { + /* NTB Port is configured as DSD/USP */ + ntb->conn_type = NTB_CONN_B2B; + ntb->dev_type = NTB_DEV_DSD; + } else { + device_printf(ntb->device, "Unsupported connection type: %u\n", + (ppd1 & SPR_PPD_CONN_MASK)); + return (ENXIO); + } + } + + device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x," + "features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features); + + return (0); +} + static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb) { @@ -1840,6 +2034,39 @@ intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb) return (rc); } +static int +intel_ntb_xeon_gen4_init_dev(struct ntb_softc *ntb) +{ + int rc; + uint16_t lnkctl; + + ntb->spad_count = XEON_GEN4_SPAD_COUNT; + ntb->db_count = XEON_GEN4_DB_COUNT; + ntb->db_link_mask = XEON_GEN4_DB_LINK_BIT; + ntb->db_vec_count = XEON_GEN4_DB_MSIX_VECTOR_COUNT; + ntb->db_vec_shift = XEON_GEN4_DB_MSIX_VECTOR_SHIFT; + + if (intel_ntb_detect_xeon_gen4_cfg(ntb) != 0) + return (ENXIO); + + ntb->reg = &xeon_gen4_reg; + ntb->self_reg = &xeon_gen4_pri_reg; + ntb->peer_reg = &xeon_gen4_b2b_reg; + ntb->xlat_reg = &xeon_gen4_sec_xlat; + + ntb->db_valid_mask = (1ULL << ntb->db_count) - 1; + xeon_gen4_setup_b2b_mw(ntb); + + /* init link setup */ + lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL); + lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE; + intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl); + + /* Setup Interrupt */ + rc = intel_ntb_xeon_gen4_init_isr(ntb); + return (rc); +} + static int intel_ntb_atom_init_dev(struct ntb_softc *ntb) { @@ -2181,6 +2408,47 @@ xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb) return (0); } +static int +xeon_gen4_setup_b2b_mw(struct ntb_softc *ntb) +{ + uint32_t embarsz, imbarsz; + + /* IMBAR23SZ should be equal to EMBAR23SZ */ + imbarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_IMBAR1SZ, 1); + embarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_EMBAR1SZ, 1); + if (embarsz != imbarsz) { + device_printf(ntb->device, + "IMBAR23SZ (%u) should be equal to EMBAR23SZ (%u)\n", + imbarsz, embarsz); + return (EIO); + } + /* IMBAR45SZ should be equal to EMBAR45SZ */ + imbarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_IMBAR2SZ, 1); + embarsz = pci_read_config(ntb->device, XEON_GEN4_CFG_REG_EMBAR2SZ, 1); + if (embarsz != imbarsz) { + device_printf(ntb->device, + "IMBAR45SZ (%u) should be equal to EMBAR45SZ (%u)\n", + imbarsz, embarsz); + return (EIO); + } + + /* Client will provide the incoming IMBARXBASE, zero it for now */ + intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR1XBASE, 0); + intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR2XBASE, 0); + + /* + * If the value in IMBARXLIMIT is set equal to the value in IMBARXBASE, + * the local memory window exposure from EMBAR is disabled. + * Note: It is needed to avoid malicious access. + */ + intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR1XLIMIT, 0); + intel_ntb_reg_write(8, XEON_GEN4_REG_IMBAR2XLIMIT, 0); + + /* EMBARXLIMIT & EMBARXBASE are gone for gen4, noop here */ + + return (0); +} + static inline bool _xeon_link_is_up(struct ntb_softc *ntb) { @@ -2194,7 +2462,9 @@ static inline bool link_is_up(struct ntb_softc *ntb) { - if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3) + if (ntb->type == NTB_XEON_GEN1 || + ntb->type == NTB_XEON_GEN3 || + ntb->type == NTB_XEON_GEN4) return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); @@ -2326,6 +2596,52 @@ intel_ntb_peer_port_idx(device_t dev, int port) return (0); } +static int +intel_ntb4_link_enable(device_t dev, enum ntb_speed speed __unused, + enum ntb_width width __unused) +{ + struct ntb_softc *ntb = device_get_softc(dev); + uint32_t cntl, ppd0, ltr; + uint16_t lnkctl; + + if (!HAS_FEATURE(ntb, NTB_LTR_BAD)) { + /* Setup active snoop LTR values */ + ltr = NTB_LTR_ACTIVE_REQMNT | NTB_LTR_ACTIVE_VAL | NTB_LTR_ACTIVE_LATSCALE; + /* Setup active non-snoop values */ + ltr = (ltr << NTB_LTR_NS_SHIFT) | ltr; + intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_ACTIVE, ltr); + + /* Setup idle snoop LTR values */ + ltr = NTB_LTR_IDLE_VAL | NTB_LTR_IDLE_LATSCALE | NTB_LTR_IDLE_REQMNT; + /* Setup idle non-snoop values */ + ltr = (ltr << NTB_LTR_NS_SHIFT) | ltr; + intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_IDLE, ltr); + + /* setup PCIe LTR to active */ + intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_SWSEL, NTB_LTR_SWSEL_ACTIVE); + } + + cntl = NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP; + cntl |= NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP; + intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); + + lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL); + lnkctl &= ~GEN4_LINK_CTRL_LINK_DISABLE; + intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl); + + /* start link training in PPD0 */ + ppd0 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD0); + ppd0 |= GEN4_PPD_LINKTRN; + intel_ntb_reg_write(4, XEON_GEN4_REG_PPD0, ppd0); + + /* make sure link training has started */ + ppd0 = intel_ntb_reg_read(4, XEON_GEN4_REG_PPD0); + if (!(ppd0 & GEN4_PPD_LINKTRN)) + intel_ntb_printf(2, "Link is not training\n"); + + return (0); +} + static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, enum ntb_width width __unused) @@ -2335,6 +2651,9 @@ intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, intel_ntb_printf(2, "%s\n", __func__); + if (ntb->type == NTB_XEON_GEN4) + return (intel_ntb4_link_enable(dev, speed, width)); + if (ntb->type == NTB_ATOM) { pci_write_config(ntb->device, NTB_PPD_OFFSET, ntb->ppd | ATOM_PPD_INIT_LINK, 4); @@ -2356,6 +2675,30 @@ intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, return (0); } +static int +intel_ntb4_link_disable(device_t dev) +{ + struct ntb_softc *ntb = device_get_softc(dev); + uint32_t cntl; + uint16_t lnkctl; + + /* clear the snoop bits */ + cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); + cntl &= ~(NTB_CTL_E2I_BAR23_SNOOP | NTB_CTL_I2E_BAR23_SNOOP); + cntl &= ~(NTB_CTL_E2I_BAR45_SNOOP | NTB_CTL_I2E_BAR45_SNOOP); + intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); + + lnkctl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL); + lnkctl |= GEN4_LINK_CTRL_LINK_DISABLE; + intel_ntb_reg_write(2, XEON_GEN4_REG_LINK_CTRL, lnkctl); + + /* set LTR to idle */ + if (!HAS_FEATURE(ntb, NTB_LTR_BAD)) + intel_ntb_reg_write(4, XEON_GEN4_REG_EXT_LTR_SWSEL, NTB_LTR_SWSEL_IDLE); + + return (0); +} + static int intel_ntb_link_disable(device_t dev) { @@ -2364,6 +2707,9 @@ intel_ntb_link_disable(device_t dev) intel_ntb_printf(2, "%s\n", __func__); + if (ntb->type == NTB_XEON_GEN4) + return (intel_ntb4_link_disable(dev)); + if (ntb->conn_type == NTB_CONN_TRANSPARENT) { ntb_link_event(dev); return (0); @@ -2393,6 +2739,11 @@ intel_ntb_link_enabled(device_t dev) if (ntb->conn_type == NTB_CONN_TRANSPARENT) return (true); + if (ntb->type == NTB_XEON_GEN4) { + cntl = intel_ntb_reg_read(2, XEON_GEN4_REG_LINK_CTRL); + return ((cntl & GEN4_LINK_CTRL_LINK_DISABLE) == 0); + } + cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); } @@ -2445,45 +2796,88 @@ retry: * Polls the HW link status register(s); returns true if something has changed. */ static bool -intel_ntb_poll_link(struct ntb_softc *ntb) +intel_ntb_atom_poll_link(struct ntb_softc *ntb) { uint32_t ntb_cntl; - uint16_t reg_val; - if (ntb->type == NTB_ATOM) { - ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); - if (ntb_cntl == ntb->ntb_ctl) - return (false); + ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); + if (ntb_cntl == ntb->ntb_ctl) + return (false); - ntb->ntb_ctl = ntb_cntl; - ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); - } else { - if (ntb->type == NTB_XEON_GEN1) - db_iowrite_raw(ntb, ntb->self_reg->db_bell, - ntb->db_link_mask); + ntb->ntb_ctl = ntb_cntl; + ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); + return (true); +} + +static bool +intel_ntb_xeon_gen1_poll_link(struct ntb_softc *ntb) +{ + uint16_t reg_val; - reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); - if (reg_val == ntb->lnk_sta) - return (false); + if (ntb->type == NTB_XEON_GEN1) + db_iowrite_raw(ntb, ntb->self_reg->db_bell, + ntb->db_link_mask); - ntb->lnk_sta = reg_val; + reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); + if (reg_val == ntb->lnk_sta) + return (false); - if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { - if (_xeon_link_is_up(ntb)) { - if (!ntb->peer_msix_good) { - callout_reset(&ntb->peer_msix_work, 0, - intel_ntb_exchange_msix, ntb); - return (false); - } - } else { - ntb->peer_msix_good = false; - ntb->peer_msix_done = false; + ntb->lnk_sta = reg_val; + + if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { + if (_xeon_link_is_up(ntb)) { + if (!ntb->peer_msix_good) { + callout_reset(&ntb->peer_msix_work, 0, + intel_ntb_exchange_msix, ntb); + return (false); } + } else { + ntb->peer_msix_good = false; + ntb->peer_msix_done = false; } } return (true); } +static bool +intel_ntb_xeon_gen4_poll_link(struct ntb_softc *ntb) +{ + uint16_t reg_val; + + /* + * We need to write to DLLSCS bit in the SLOTSTS before we + * can clear the hardware link interrupt on ICX NTB. + */ + intel_ntb_reg_write(2, XEON_GEN4_REG_SLOTSTS, GEN4_SLOTSTS_DLLSCS); + db_iowrite_raw(ntb, ntb->self_reg->db_clear, ntb->db_link_mask); + + reg_val = intel_ntb_reg_read(2, ntb->reg->lnk_sta); + if (reg_val == ntb->lnk_sta) + return (false); + + ntb->lnk_sta = reg_val; + return (true); +} + +static bool +intel_ntb_poll_link(struct ntb_softc *ntb) +{ + bool val; + + switch(ntb->type) { + case NTB_ATOM: + val = intel_ntb_atom_poll_link(ntb); + break; + case NTB_XEON_GEN4: + val = intel_ntb_xeon_gen4_poll_link(ntb); + break; + default: + val = intel_ntb_xeon_gen1_poll_link(ntb); + break; + } + return (val); +} + static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *ntb) { @@ -3231,7 +3625,7 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) limit = 0; if (bar_is_64bit(ntb, bar_num)) { - if (ntb->type == NTB_XEON_GEN3) + if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) base = addr; else base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; @@ -3259,7 +3653,7 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) } } else { /* Configure 32-bit (split) BAR MW */ - if (ntb->type == NTB_XEON_GEN3) + if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) return (EIO); if ((addr & UINT32_MAX) != addr) @@ -3376,7 +3770,7 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bits) return; } - if (ntb->type == NTB_XEON_GEN3) { + if (ntb->type == NTB_XEON_GEN3 || ntb->type == NTB_XEON_GEN4) { while (bits != 0) { db = ffsll(bits); diff --git a/sys/dev/ntb/ntb_hw/ntb_hw_intel.h b/sys/dev/ntb/ntb_hw/ntb_hw_intel.h index c861b77743e3..9f9cadaa4d47 100644 --- a/sys/dev/ntb/ntb_hw/ntb_hw_intel.h +++ b/sys/dev/ntb/ntb_hw/ntb_hw_intel.h @@ -250,4 +250,119 @@ #define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1) #define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1) +/* Xeon ICX/SPR NTB register definitions */ + +/* CFG Space */ +#define XEON_GEN4_CFG_REG_BAR0BASE 0x0010 +#define XEON_GEN4_CFG_REG_BAR1BASE 0x0018 +#define XEON_GEN4_CFG_REG_BAR2BASE 0x0020 +#define XEON_GEN4_CFG_REG_IMBAR1SZ 0x00c4 +#define XEON_GEN4_CFG_REG_IMBAR2SZ 0x00c5 +#define XEON_GEN4_CFG_REG_EMBAR1SZ 0x00c6 +#define XEON_GEN4_CFG_REG_EMBAR2SZ 0x00c7 +#define XEON_GEN4_CFG_REG_DEVCTRL 0x0048 +#define XEON_GEN4_CFG_REG_DEVSTS 0x004a +#define XEON_GEN4_CFG_REG_UNCERRSTS 0x0104 +#define XEON_GEN4_CFG_REG_CORERRSTS 0x0110 + +/* BAR0 MMIO */ +#define XEON_GEN4_REG_IMNTB_CTL 0x0000 +#define XEON_GEN4_REG_IMBAR1XBASE 0x0010 +#define XEON_GEN4_REG_IMBAR1XLIMIT 0x0018 +#define XEON_GEN4_REG_IMBAR2XBASE 0x0020 +#define XEON_GEN4_REG_IMBAR2XLIMIT 0x0028 +#define XEON_GEN4_REG_IMINT_STATUS 0x0040 +#define XEON_GEN4_REG_IMINT_DISABLE 0x0048 +#define XEON_GEN4_REG_INTVEC 0x0050 /* 0-32 vecs */ +#define XEON_GEN4_REG_IMSPAD 0x0080 /* 0-15 SPADs */ +#define XEON_GEN4_REG_IMDOORBELL 0x0100 /* 0-31 doorbells */ + +/* + * External EndPoint Configuration Registers + * These are located within BAR0 of the internal endpoint. + */ +#define XEON_GEN4_REG_EXT_BAR1BASE 0x3018 +#define XEON_GEN4_REG_EXT_BAR2BASE 0x3020 +#define XEON_GEN4_REG_EXT_LTR_SWSEL 0x30ec +#define XEON_GEN4_REG_EXT_LTR_ACTIVE 0x30f0 +#define XEON_GEN4_REG_EXT_LTR_IDLE 0x30f4 + +#define XEON_GEN4_REG_EMSPAD 0x8080 /* 32K + SPAD_offset */ + +/* note, link status is now in MMIO and not config space for NTB */ +#define XEON_GEN4_REG_LINK_CTRL 0xb050 +#define XEON_GEN4_REG_LINK_STATUS 0xb052 +#define XEON_GEN4_REG_SLOTSTS 0xb05a +#define XEON_GEN4_REG_PPD0 0xb0d4 +#define XEON_GEN4_REG_PPD1 0xb4c0 +#define XEON_GEN4_REG_LTSSMSTATEJMP 0xf040 + +/* Supported PCI device revision range for ICX */ +#define PCI_DEV_REV_ICX_MIN 0x2 +#define PCI_DEV_REV_ICX_MAX 0xF + +#define XEON_GEN4_DB_COUNT 32 +#define XEON_GEN4_DB_LINK 32 +#define XEON_GEN4_DB_LINK_BIT (1ULL << XEON_GEN4_DB_LINK) +#define XEON_GEN4_DB_MSIX_VECTOR_COUNT 33 +#define XEON_GEN4_DB_MSIX_VECTOR_SHIFT 1 +#define XEON_GEN4_DB_TOTAL_SHIFT 33 +#define XEON_GEN4_SPAD_COUNT 16 + +/* NTBCTL field */ +#define NTB_CTL_E2I_BAR23_SNOOP 0x000004 +#define NTB_CTL_E2I_BAR23_NOSNOOP 0x000008 +#define NTB_CTL_I2E_BAR23_SNOOP 0x000010 +#define NTB_CTL_I2E_BAR23_NOSNOOP 0x000020 +#define NTB_CTL_E2I_BAR45_SNOOP 0x000040 +#define NTB_CTL_E2I_BAR45_NOSNOO 0x000080 +#define NTB_CTL_I2E_BAR45_SNOOP 0x000100 +#define NTB_CTL_I2E_BAR45_NOSNOOP 0x000200 +#define NTB_CTL_BUSNO_DIS_INC 0x000400 +#define NTB_CTL_LINK_DOWN 0x010000 + +#define NTB_SJC_FORCEDETECT 0x000004 + +/* PPD field */ +#define GEN4_PPD_CLEAR_TRN 0x0001 +#define GEN4_PPD_LINKTRN 0x0008 +#define GEN4_PPD_CONN_MASK 0x0300 +#define SPR_PPD_CONN_MASK 0x0700 +#define GEN4_PPD_CONN_B2B 0x0200 +#define GEN4_PPD_DEV_MASK 0x1000 +#define GEN4_PPD_DEV_DSD 0x1000 +#define GEN4_PPD_DEV_USD 0x0000 +#define SPR_PPD_DEV_MASK 0x4000 +#define SPR_PPD_DEV_DSD 0x4000 +#define SPR_PPD_DEV_USD 0x0000 + +#define GEN4_LINK_CTRL_LINK_DISABLE 0x0010 +#define GEN4_SLOTSTS_DLLSCS 0x100 + +#define GEN4_PPD_TOPO_MASK \ + (GEN4_PPD_CONN_MASK | GEN4_PPD_DEV_MASK) +#define GEN4_PPD_TOPO_B2B_USD \ + (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_USD) +#define GEN4_PPD_TOPO_B2B_DSD \ + (GEN4_PPD_CONN_B2B | GEN4_PPD_DEV_DSD) + +#define SPR_PPD_TOPO_MASK \ + (SPR_PPD_CONN_MASK | SPR_PPD_DEV_MASK) +#define SPR_PPD_TOPO_B2B_USD \ + (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_USD) +#define SPR_PPD_TOPO_B2B_DSD \ + (GEN4_PPD_CONN_B2B | SPR_PPD_DEV_DSD) + +/* LTR field */ +#define NTB_LTR_SWSEL_ACTIVE 0x0 +#define NTB_LTR_SWSEL_IDLE 0x1 + +#define NTB_LTR_NS_SHIFT 16 +#define NTB_LTR_ACTIVE_VAL 0x0000 /* 0 us */ +#define NTB_LTR_ACTIVE_LATSCALE 0x0800 /* 1us scale */ +#define NTB_LTR_ACTIVE_REQMNT 0x8000 /* snoop req enable */ + +#define NTB_LTR_IDLE_VAL 0x0258 /* 600 us */ +#define NTB_LTR_IDLE_LATSCALE 0x0800 /* 1us scale */ +#define NTB_LTR_IDLE_REQMNT 0x8000 /* snoop req enable */ #endif /* _NTB_REGS_H_ */
