From: Farhan Ali <[email protected]> Add an s390x specific handler for vfio error notifier. For s390x pci devices, we have platform specific error information. We need to retrieve this error information for passthrough devices. This is done via a VFIO_DEVICE_FEATURE ioctl which exposes that information.
Once this error information is retrieved we can then inject an error into the guest, and let the guest drive the recovery. Signed-off-by: Farhan Ali <[email protected]> Reviewed-by: Matthew Rosato <[email protected]> Link: https://lore.kernel.org/qemu-devel/[email protected] Signed-off-by: Eric Farman <[email protected]> --- hw/s390x/s390-pci-bus.c | 6 ++ hw/s390x/s390-pci-vfio-stubs.c | 6 ++ hw/s390x/s390-pci-vfio.c | 123 +++++++++++++++++++++++++++++++ include/hw/s390x/s390-pci-bus.h | 1 + include/hw/s390x/s390-pci-vfio.h | 1 + 5 files changed, 137 insertions(+) diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c index b8bb663113..53ed79d17a 100644 --- a/hw/s390x/s390-pci-bus.c +++ b/hw/s390x/s390-pci-bus.c @@ -1185,6 +1185,7 @@ static void s390_pcihost_plug(const HotplugHandler *hotplug_dev, DeviceState *de PCIDevice *pdev = NULL; S390PCIBusDevice *pbdev = NULL; bool auto_pbdev = false; + Error *local_err = NULL; int rc; if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) { @@ -1275,6 +1276,11 @@ static void s390_pcihost_plug(const HotplugHandler *hotplug_dev, DeviceState *de pbdev->dma_limit = s390_pci_start_dma_count(s, pbdev); /* Fill in CLP information passed via the vfio region */ s390_pci_get_clp_info(pbdev); + /* Setup error handler for error recovery */ + if (!s390_pci_setup_err_handler(pbdev, &local_err)) { + warn_report_err(local_err); + } + if (!pbdev->interp) { /* Do vfio passthrough but intercept for I/O */ pbdev->fh |= FH_SHM_VFIO; diff --git a/hw/s390x/s390-pci-vfio-stubs.c b/hw/s390x/s390-pci-vfio-stubs.c index d9882b7aad..9fc84ca135 100644 --- a/hw/s390x/s390-pci-vfio-stubs.c +++ b/hw/s390x/s390-pci-vfio-stubs.c @@ -30,3 +30,9 @@ bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh) void s390_pci_get_clp_info(S390PCIBusDevice *pbdev) { } + +bool s390_pci_setup_err_handler(S390PCIBusDevice *pbdev, Error **errp) +{ + error_setg(errp, "VFIO not available, cannot setup error handler"); + return false; +} diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c index a035ba5b2c..e65e2be473 100644 --- a/hw/s390x/s390-pci-vfio.c +++ b/hw/s390x/s390-pci-vfio.c @@ -10,6 +10,7 @@ */ #include "qemu/osdep.h" +#include "qemu/error-report.h" #include <sys/ioctl.h> #include <linux/vfio.h> @@ -105,6 +106,80 @@ void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt) } } +static int s390_pci_get_feature_err(VFIOPCIDevice *vfio_pci, + PciCcdfErr *ccdf, + uint32_t ccdf_err_length, + Error **errp) +{ + ERRP_GUARD(); + int ret; + size_t total_size; + struct vfio_device_feature_zpci_err *err; + g_autofree void *buf = NULL; + g_autofree struct vfio_device_feature *feature = NULL; + + total_size = sizeof(*feature) + sizeof(*err); + feature = g_malloc(total_size); + feature->argsz = total_size; + feature->flags = VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_ZPCI_ERROR; + + buf = g_malloc(ccdf_err_length); + err = (void *)feature->data; + err->data = (uint64_t)buf; + ret = vfio_device_get_feature(&vfio_pci->vbasedev, feature); + + if (ret) { + if (ret == -ENOMSG) { + return 0; + } + error_setg(errp, "Failed feature get VFIO_DEVICE_FEATURE_ZPCI_ERROR" + " (rc=%d)", ret); + return ret; + } + + memcpy(ccdf, buf, ccdf_err_length); + + return 1; +} + +static void s390_pci_err_handler(void *opaque) +{ + VFIOPCIDevice *vfio_pci; + S390PCIBusDevice *pbdev; + Error *local_err = NULL; + PciCcdfErr ccdf; + int ret = 0; + + vfio_pci = opaque; + if (!event_notifier_test_and_clear(&vfio_pci->err_notifier)) { + return; + } + + pbdev = s390_pci_find_dev_by_pci(s390_get_phb(), PCI_DEVICE(vfio_pci)); + + if (!pbdev) { + error_report("No matching zpci device found"); + return; + } + pbdev->state = ZPCI_FS_ERROR; + + do { + ret = s390_pci_get_feature_err(vfio_pci, &ccdf, + pbdev->ccdf_err_length, &local_err); + if (ret < 0) { + error_report_err(local_err); + break; + } + + if (ret > 0) { + s390_pci_generate_error_event(ccdf.pec, pbdev->fh, pbdev->fid, + ccdf.faddr, ccdf.e); + } + } while (ret > 0); + + return; +} + static void s390_pci_read_base(S390PCIBusDevice *pbdev, struct vfio_device_info *info) { @@ -134,6 +209,10 @@ static void s390_pci_read_base(S390PCIBusDevice *pbdev, /* Store function type separately for type-specific behavior */ pbdev->pft = cap->pft; + if (hdr->version >= 3) { + pbdev->ccdf_err_length = cap->ccdf_err_length; + } + /* * If the device is a passthrough ISM device, disallow relaxed * translation. @@ -371,3 +450,47 @@ void s390_pci_get_clp_info(S390PCIBusDevice *pbdev) s390_pci_read_util(pbdev, info); s390_pci_read_pfip(pbdev, info); } + +bool s390_pci_setup_err_handler(S390PCIBusDevice *pbdev, Error **errp) +{ + int ret; + int32_t fd; + VFIOPCIDevice *vfio_pci = VFIO_PCI_DEVICE(pbdev->pdev); + uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature), + sizeof(uint64_t))] = {}; + struct vfio_device_feature *feature = (struct vfio_device_feature *)buf; + + feature->argsz = sizeof(buf); + feature->flags = VFIO_DEVICE_FEATURE_PROBE | VFIO_DEVICE_FEATURE_ZPCI_ERROR; + + ret = vfio_device_get_feature(&vfio_pci->vbasedev, feature); + + if (ret != 0) { + if (ret == -ENOTTY) { + error_setg(errp, "Automated error recovery unavailable for device"); + } else { + error_setg(errp, + "Failed to probe for VFIO_DEVICE_FEATURE_ZPCI_ERROR (ret=%d)", + ret); + } + return false; + } + + if (sizeof(PciCcdfErr) != pbdev->ccdf_err_length) { + error_setg(errp, + "CCDF size mismatch expected size=%zu, provided size=%d", + sizeof(PciCcdfErr), pbdev->ccdf_err_length); + return false; + } + + if (!vfio_pci->pci_aer) { + error_setg(errp, "%s: VFIO error notifier not registered for device", + vfio_pci->vbasedev.name); + return false; + } + + fd = event_notifier_get_fd(&vfio_pci->err_notifier); + qemu_set_fd_handler(fd, s390_pci_err_handler, NULL, vfio_pci); + + return true; +} diff --git a/include/hw/s390x/s390-pci-bus.h b/include/hw/s390x/s390-pci-bus.h index 71f956700b..b9c9926910 100644 --- a/include/hw/s390x/s390-pci-bus.h +++ b/include/hw/s390x/s390-pci-bus.h @@ -374,6 +374,7 @@ struct S390PCIBusDevice { bool forwarding_assist; bool aif; bool rtr_avail; + uint32_t ccdf_err_length; QTAILQ_ENTRY(S390PCIBusDevice) link; }; diff --git a/include/hw/s390x/s390-pci-vfio.h b/include/hw/s390x/s390-pci-vfio.h index f7d6149daf..c7886b63ea 100644 --- a/include/hw/s390x/s390-pci-vfio.h +++ b/include/hw/s390x/s390-pci-vfio.h @@ -20,5 +20,6 @@ S390PCIDMACount *s390_pci_start_dma_count(S390pciState *s, void s390_pci_end_dma_count(S390pciState *s, S390PCIDMACount *cnt); bool s390_pci_get_host_fh(S390PCIBusDevice *pbdev, uint32_t *fh); void s390_pci_get_clp_info(S390PCIBusDevice *pbdev); +bool s390_pci_setup_err_handler(S390PCIBusDevice *pbdev, Error **errp); #endif -- 2.56.0
