s390_pci_iommu_disable() destroys the I/O TLB and unpins the host's mappings, but it doesn't reset the available DMA slot counter back to the VFIO-advertised limit. This is problematic when an IOMMU is disabled and re-enabled while reusing the container (e.g. during a guest reboot), which may cause the RPCIT path to fail with DMA slot exhaustion errors, manifesting as unexplained ENOMEMs in the guest.
Fix it by refactoring s390_pci_update_dma_avail() to take a pointer to S390PCIBusDevice instead of the plain file descriptor. Call it in s390_pci_iommu_disable() after the host mappings have been unpinned. Further print a warning if the VFIO-advertised counter doesn't match the in-use shadow counter, to reduce the likelihood of introducing new similar DMA accounting bugs in the future. Cc: [email protected] Fixes: 37fa32de70 ("s390x/pci: Honor DMA limits set by vfio") Signed-off-by: Omar Elghoul <[email protected]> --- hw/s390x/s390-pci-bus.c | 19 +++++++++++++++++++ hw/s390x/s390-pci-vfio-stubs.c | 2 +- hw/s390x/s390-pci-vfio.c | 18 ++++++++++++------ include/hw/s390x/s390-pci-vfio.h | 2 +- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index 2eb4e8cec4..0da9f67f42 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -762,6 +762,8 @@ void s390_pci_iommu_direct_map_enable(S390PCIIOMMU *iommu) void s390_pci_iommu_disable(S390PCIIOMMU *iommu) { + unsigned int dma_avail; + iommu->enabled = false; g_hash_table_remove_all(iommu->iotlb); if (iommu->dm_mr) { @@ -774,6 +776,23 @@ void s390_pci_iommu_disable(S390PCIIOMMU *iommu) MEMORY_REGION(&iommu->iommu_mr)); object_unparent(OBJECT(&iommu->iommu_mr)); } + + /* + * update shadow counter from vfio after having unpinned all host mappings. + * if this value mismatches our previous shadow count, then we likely have + * DMA remapping, unmapping, or accounting bugs + */ + if (iommu->pbdev && s390_pci_update_dma_avail(iommu->pbdev, &dma_avail)) { + if (iommu->max_dma_limit != dma_avail) { + warn_report("zpci: DMA limit %u from vfio != %" PRIu64 " shadowed", + dma_avail, iommu->max_dma_limit); + iommu->max_dma_limit = dma_avail; + } + + if (iommu->dma_limit) { + iommu->dma_limit->avail = dma_avail; + } + } } static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn) diff --git a/hw/s390x/s390-pci-vfio-stubs.c b/hw/s390x/s390-pci-vfio-stubs.c index d9882b7aad..da41c75602 100644 --- a/hw/s390x/s390-pci-vfio-stubs.c +++ b/hw/s390x/s390-pci-vfio-stubs.c @@ -7,7 +7,7 @@ #include "qemu/osdep.h" #include "hw/s390x/s390-pci-vfio.h" -bool s390_pci_update_dma_avail(int fd, unsigned int *avail) +bool s390_pci_update_dma_avail(S390PCIBusDevice *pbdev, unsigned int *avail) { return false; } diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c index db6de00bd2..caa3d88bc1 100644 --- a/hw/s390x/s390-pci-vfio.c +++ b/hw/s390x/s390-pci-vfio.c @@ -29,13 +29,23 @@ * limiting DMA requests, false otherwise. The current available count read * from vfio is returned in avail. */ -bool s390_pci_update_dma_avail(int fd, unsigned int *avail) +bool s390_pci_update_dma_avail(S390PCIBusDevice *pbdev, unsigned int *avail) { + VFIOPCIDevice *vpdev; uint32_t argsz = sizeof(struct vfio_iommu_type1_info); g_autofree struct vfio_iommu_type1_info *info = g_malloc0(argsz); + int fd; + assert(pbdev); assert(avail); + vpdev = VFIO_PCI_DEVICE(pbdev->pdev); + if (!vpdev || !vpdev->vbasedev.group) { + return false; + } + + fd = vpdev->vbasedev.group->container->fd; + /* * If the specified argsz is not large enough to contain all capabilities * it will be updated upon return from the ioctl. Retry until we have @@ -68,16 +78,12 @@ S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s, assert(vpdev); - if (!vpdev->vbasedev.group) { + if (!s390_pci_update_dma_avail(pbdev, &avail)) { return NULL; } id = vpdev->vbasedev.group->container->fd; - if (!s390_pci_update_dma_avail(id, &avail)) { - return NULL; - } - QTAILQ_FOREACH(cnt, &s->zpci_dma_limit, link) { if (cnt->id == id) { cnt->users++; diff --git a/include/hw/s390x/s390-pci-vfio.h b/include/hw/s390x/s390-pci-vfio.h index f7d6149daf..c11d5781d2 100644 --- a/include/hw/s390x/s390-pci-vfio.h +++ b/include/hw/s390x/s390-pci-vfio.h @@ -14,7 +14,7 @@ #include "hw/s390x/s390-pci-bus.h" -bool s390_pci_update_dma_avail(int fd, unsigned int *avail); +bool s390_pci_update_dma_avail(S390PCIBusDevice *pbdev, unsigned int *avail); S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s, S390PCIBusDevice *pbdev); void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt); -- 2.56.0
