Author: hselasky
Date: Tue Mar 14 15:15:37 2017
New Revision: 315247
URL: https://svnweb.freebsd.org/changeset/base/315247
Log:
MFC r310806:
Add MSIX rewrite table quirk for use with VMs.
This patch solves IRQ generation problems using the mlx5en(4) driver
with xenserver v6.5.0 in SRIOV and PCI-passthrough modes.
Until further the hw.pci.msix_rewrite_table quirk must be set manually
in /boot/loader.conf .
Reviewed by: jhb @
Sponsored by: Mellanox Technologies
Modified:
stable/10/sys/dev/pci/pci.c
Directory Properties:
stable/10/ (props changed)
Modified: stable/10/sys/dev/pci/pci.c
==============================================================================
--- stable/10/sys/dev/pci/pci.c Tue Mar 14 15:12:58 2017 (r315246)
+++ stable/10/sys/dev/pci/pci.c Tue Mar 14 15:15:37 2017 (r315247)
@@ -351,6 +351,11 @@ TUNABLE_INT("hw.pci.enable_msix", &pci_d
SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RW, &pci_do_msix, 1,
"Enable support for MSI-X interrupts");
+static int pci_msix_rewrite_table = 0;
+SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN,
+ &pci_msix_rewrite_table, 0,
+ "Rewrite entire MSI-X table when updating MSI-X entries");
+
static int pci_honor_msi_blacklist = 1;
TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist);
SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RD,
@@ -1375,11 +1380,10 @@ pci_find_extcap_method(device_t dev, dev
/*
* Support for MSI-X message interrupts.
*/
-void
-pci_enable_msix_method(device_t dev, device_t child, u_int index,
- uint64_t address, uint32_t data)
+static void
+pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t
data)
{
- struct pci_devinfo *dinfo = device_get_ivars(child);
+ struct pci_devinfo *dinfo = device_get_ivars(dev);
struct pcicfg_msix *msix = &dinfo->cfg.msix;
uint32_t offset;
@@ -1388,6 +1392,31 @@ pci_enable_msix_method(device_t dev, dev
bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
bus_write_4(msix->msix_table_res, offset + 8, data);
+}
+
+void
+pci_enable_msix_method(device_t dev, device_t child, u_int index,
+ uint64_t address, uint32_t data)
+{
+
+ if (pci_msix_rewrite_table) {
+ struct pci_devinfo *dinfo = device_get_ivars(child);
+ struct pcicfg_msix *msix = &dinfo->cfg.msix;
+
+ /*
+ * Some VM hosts require MSIX to be disabled in the
+ * control register before updating the MSIX table
+ * entries are allowed. It is not enough to only
+ * disable MSIX while updating a single entry. MSIX
+ * must be disabled while updating all entries in the
+ * table.
+ */
+ pci_write_config(child,
+ msix->msix_location + PCIR_MSIX_CTRL,
+ msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2);
+ pci_resume_msix(child);
+ } else
+ pci_write_msix_entry(child, index, address, data);
/* Enable MSI -> HT mapping. */
pci_ht_map_msi(child, address);
@@ -1463,7 +1492,8 @@ pci_resume_msix(device_t dev)
if (mte->mte_vector == 0 || mte->mte_handlers == 0)
continue;
mv = &msix->msix_vectors[mte->mte_vector - 1];
- pci_enable_msix(dev, i, mv->mv_address, mv->mv_data);
+ pci_write_msix_entry(dev, i, mv->mv_address,
+ mv->mv_data);
pci_unmask_msix(dev, i);
}
}
@@ -3980,12 +4010,20 @@ pci_setup_intr(device_t dev, device_t ch
mv->mv_address = addr;
mv->mv_data = data;
}
- if (mte->mte_handlers == 0) {
+
+ /*
+ * The MSIX table entry must be made valid by
+ * incrementing the mte_handlers before
+ * calling pci_enable_msix() and
+ * pci_resume_msix(). Else the MSIX rewrite
+ * table quirk will not work as expected.
+ */
+ mte->mte_handlers++;
+ if (mte->mte_handlers == 1) {
pci_enable_msix(child, rid - 1, mv->mv_address,
mv->mv_data);
pci_unmask_msix(child, rid - 1);
}
- mte->mte_handlers++;
}
/*
_______________________________________________
svn-src-stable-10@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-stable-10
To unsubscribe, send any mail to "svn-src-stable-10-unsubscr...@freebsd.org"
