On pSeries, the MSI-X table shadow in QEMU is not sourced from the
physical device. It is populated by sPAPR RTAS through the
ibm,change-msi call path:
ibm,change-msi
-> spapr_msi_setmsg()
-> msix_set_message() /* writes each shadow entry */
EEH recovery does not necessarily cause the guest to re-issue
ibm,change-msi. The guest VFIO PCI driver restores interrupt delivery
by writing MSI-X Enable = 1 directly via config space. QEMU then
dispatches through:
vfio_msix_enable()
-> vfio_msix_vector_do_use() /* re-arms KVM irqfd from shadow */
The existing EEH pre-reset helper calls msix_reset() after clearing
MSI-X Enable. msix_reset() zeroes every shadow entry. Without the
shadow, vfio_msix_vector_do_use() cannot reconstruct the KVM irqfd
routes and device interrupts do not recover after EEH.
Fix the pre-reset helper:
1. Remove the msix_reset() call. The shadow must survive the PE reset
so that interrupt delivery can be restored when the guest re-enables
MSI-X without repeating ibm,change-msi.
2. Read the MSI-X flags from the cached pdev->config shadow instead of
calling pci_host_config_read_common(). The PE may already be frozen
when EEH pre-reset runs; reading from physical config space could
fault.
3. Rename the helpers from clear_dev_msix / clear_bus_msix to
prepare_dev / prepare_bus to reflect that they no longer clear any
state, and fix the pre-existing tab-indented bodies to use spaces
consistently with the surrounding code.
4. Switch the VFIO device type check from the "vfio-pci" string literal
to TYPE_VFIO_PCI_DEVICE, matching the convention used by other
non-VFIO source files that access VFIOPCIDevice (e.g.
hw/s390x/s390-pci-vfio.c). Add the corresponding #include for
hw/vfio/pci.h.
The guest recovery sequence after this fix:
EEH pre-reset:
MSI-X Enable -> 0 (pci_host_config_write_common)
-> vfio_msix_disable()
-> vfio_msix_vector_release() for each vector
-> KVM irqfd routes removed
-> VFIO MSIX_IRQ_INDEX disabled
shadow entries unchanged
PE reset + PE configure
guest slot_reset() / PCI state restore:
MSI-X Enable -> 1 (config-space write)
-> vfio_msix_enable()
-> vfio_msix_vector_do_use() for each vector
-> shadow entries present -> KVM irqfd routes re-armed
interrupts recover
Signed-off-by: Narayana Murty N <[email protected]>
---
hw/ppc/spapr_pci_vfio.c | 56 ++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 23 deletions(-)
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index a748a0bf4c..1fa8847031 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -25,6 +25,7 @@
#include "hw/pci/msix.h"
#include "hw/pci/pci_device.h"
#include "hw/vfio/vfio-container-legacy.h"
+#include "hw/vfio/pci.h"
#include "qemu/error-report.h"
#include CONFIG_DEVICES /* CONFIG_VFIO_PCI */
@@ -233,47 +234,56 @@ int spapr_phb_vfio_eeh_get_state(SpaprPhbState *sphb, int
*state)
return RTAS_OUT_SUCCESS;
}
-static void spapr_phb_vfio_eeh_clear_dev_msix(PCIBus *bus,
- PCIDevice *pdev,
- void *opaque)
+/*
+ * Prepare a single VFIO PCI device for an EEH PE hot or fundamental reset.
+ *
+ * On pSeries the MSI-X table shadow is populated by ibm,change-msi via
+ * spapr_msi_setmsg() -> msix_set_message(). EEH recovery does not
+ * necessarily re-issue ibm,change-msi; the guest restores MSI-X Enable
+ * directly, causing QEMU to dispatch through vfio_msix_enable() ->
+ * vfio_msix_vector_do_use(), which re-arms the KVM irqfd routes from the
+ * existing shadow entries.
+ *
+ * Therefore, msix_reset() must NOT be called here. Calling it would wipe
+ * those shadow entries and prevent interrupt delivery after EEH recovery.
+ *
+ * Instead, clear MSI-X Enable using the cached pdev->config shadow (avoiding
+ * a read from potentially frozen device config space) and write through
+ * pci_host_config_write_common() so that the VFIO config-write handler calls
+ * vfio_msix_disable(), cleanly releasing vectors and KVM irqfd routes while
+ * leaving the shadow intact.
+ */
+static void spapr_phb_vfio_eeh_prepare_dev(PCIBus *bus,
+ PCIDevice *pdev,
+ void *opaque)
{
- /* Check if the device is VFIO PCI device */
- if (!object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+ uint16_t flags;
+
+ if (!object_dynamic_cast(OBJECT(pdev), TYPE_VFIO_PCI_DEVICE)) {
return;
}
- /*
- * The MSIx table will be cleaned out by reset. We need
- * disable it so that it can be reenabled properly. Also,
- * the cached MSIx table should be cleared as it's not
- * reflecting the contents in hardware.
- */
if (msix_enabled(pdev)) {
- uint16_t flags;
-
- flags = pci_host_config_read_common(pdev,
- pdev->msix_cap + PCI_MSIX_FLAGS,
- pci_config_size(pdev), 2);
+ flags = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
flags &= ~PCI_MSIX_FLAGS_ENABLE;
pci_host_config_write_common(pdev,
pdev->msix_cap + PCI_MSIX_FLAGS,
pci_config_size(pdev), flags, 2);
}
-
- msix_reset(pdev);
}
-static void spapr_phb_vfio_eeh_clear_bus_msix(PCIBus *bus, void *opaque)
+static void spapr_phb_vfio_eeh_prepare_bus(PCIBus *bus, void *opaque)
{
- pci_for_each_device_under_bus(bus, spapr_phb_vfio_eeh_clear_dev_msix,
- NULL);
+ pci_for_each_device_under_bus(bus,
+ spapr_phb_vfio_eeh_prepare_dev,
+ NULL);
}
static void spapr_phb_vfio_eeh_pre_reset(SpaprPhbState *sphb)
{
- PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+ PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
- pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
+ pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_prepare_bus, NULL);
}
int spapr_phb_vfio_eeh_reset(SpaprPhbState *sphb, int option)
--
2.54.0
