Forward remote device's interrupts to the guest Signed-off-by: Elena Ufimtseva <elena.ufimts...@oracle.com> Signed-off-by: John G Johnson <john.g.john...@oracle.com> Signed-off-by: Jagannathan Raman <jag.ra...@oracle.com> --- include/hw/pci/pci.h | 6 +++ hw/pci/msi.c | 13 +++++- hw/pci/msix.c | 12 +++++- hw/remote/vfio-user-obj.c | 89 +++++++++++++++++++++++++++++++++++++++ hw/remote/trace-events | 1 + 5 files changed, 119 insertions(+), 2 deletions(-)
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h index 8c18f10d9d..092334d2af 100644 --- a/include/hw/pci/pci.h +++ b/include/hw/pci/pci.h @@ -128,6 +128,8 @@ typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev, typedef void PCIMapIORegionFunc(PCIDevice *pci_dev, int region_num, pcibus_t addr, pcibus_t size, int type); typedef void PCIUnregisterFunc(PCIDevice *pci_dev); +typedef void PCIMSINotify(PCIDevice *pci_dev, unsigned vector); +typedef void PCIMSIxNotify(PCIDevice *pci_dev, unsigned vector); typedef struct PCIIORegion { pcibus_t addr; /* current PCI mapping address. -1 means not mapped */ @@ -322,6 +324,10 @@ struct PCIDevice { /* Space to store MSIX table & pending bit array */ uint8_t *msix_table; uint8_t *msix_pba; + + PCIMSINotify *msi_notify; + PCIMSIxNotify *msix_notify; + /* MemoryRegion container for msix exclusive BAR setup */ MemoryRegion msix_exclusive_bar; /* Memory Regions for MSIX table and pending bit entries. */ diff --git a/hw/pci/msi.c b/hw/pci/msi.c index 47d2b0f33c..93f5e400cc 100644 --- a/hw/pci/msi.c +++ b/hw/pci/msi.c @@ -51,6 +51,8 @@ */ bool msi_nonbroken; +static void pci_msi_notify(PCIDevice *dev, unsigned int vector); + /* If we get rid of cap allocator, we won't need this. */ static inline uint8_t msi_cap_sizeof(uint16_t flags) { @@ -225,6 +227,8 @@ int msi_init(struct PCIDevice *dev, uint8_t offset, dev->msi_cap = config_offset; dev->cap_present |= QEMU_PCI_CAP_MSI; + dev->msi_notify = pci_msi_notify; + pci_set_word(dev->config + msi_flags_off(dev), flags); pci_set_word(dev->wmask + msi_flags_off(dev), PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); @@ -307,7 +311,7 @@ bool msi_is_masked(const PCIDevice *dev, unsigned int vector) return mask & (1U << vector); } -void msi_notify(PCIDevice *dev, unsigned int vector) +static void pci_msi_notify(PCIDevice *dev, unsigned int vector) { uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; @@ -332,6 +336,13 @@ void msi_notify(PCIDevice *dev, unsigned int vector) msi_send_message(dev, msg); } +void msi_notify(PCIDevice *dev, unsigned int vector) +{ + if (dev->msi_notify) { + dev->msi_notify(dev, vector); + } +} + void msi_send_message(PCIDevice *dev, MSIMessage msg) { MemTxAttrs attrs = {}; diff --git a/hw/pci/msix.c b/hw/pci/msix.c index ae9331cd0b..1c71e67f53 100644 --- a/hw/pci/msix.c +++ b/hw/pci/msix.c @@ -31,6 +31,8 @@ #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8) #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8) +static void pci_msix_notify(PCIDevice *dev, unsigned vector); + MSIMessage msix_get_message(PCIDevice *dev, unsigned vector) { uint8_t *table_entry = dev->msix_table + vector * PCI_MSIX_ENTRY_SIZE; @@ -334,6 +336,7 @@ int msix_init(struct PCIDevice *dev, unsigned short nentries, dev->msix_table = g_malloc0(table_size); dev->msix_pba = g_malloc0(pba_size); dev->msix_entry_used = g_malloc0(nentries * sizeof *dev->msix_entry_used); + dev->msix_notify = pci_msix_notify; msix_mask_all(dev, nentries); @@ -485,7 +488,7 @@ int msix_enabled(PCIDevice *dev) } /* Send an MSI-X message */ -void msix_notify(PCIDevice *dev, unsigned vector) +static void pci_msix_notify(PCIDevice *dev, unsigned vector) { MSIMessage msg; @@ -503,6 +506,13 @@ void msix_notify(PCIDevice *dev, unsigned vector) msi_send_message(dev, msg); } +void msix_notify(PCIDevice *dev, unsigned vector) +{ + if (dev->msix_notify) { + dev->msix_notify(dev, vector); + } +} + void msix_reset(PCIDevice *dev) { if (!msix_present(dev)) { diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c index bf88eac8f1..1771dba1bf 100644 --- a/hw/remote/vfio-user-obj.c +++ b/hw/remote/vfio-user-obj.c @@ -53,6 +53,8 @@ #include "hw/qdev-core.h" #include "hw/pci/pci.h" #include "qemu/timer.h" +#include "hw/pci/msi.h" +#include "hw/pci/msix.h" #define TYPE_VFU_OBJECT "x-vfio-user-server" OBJECT_DECLARE_TYPE(VfuObject, VfuObjectClass, VFU_OBJECT) @@ -106,6 +108,8 @@ struct VfuObject { int vfu_poll_fd; }; +static GHashTable *vfu_object_dev_to_ctx_table; + static void vfu_object_init_ctx(VfuObject *o, Error **errp); static void vfu_object_set_socket(Object *obj, Visitor *v, const char *name, @@ -381,6 +385,72 @@ static void vfu_object_register_bars(vfu_ctx_t *vfu_ctx, PCIDevice *pdev) } } +static int vfu_object_map_irq(PCIDevice *pci_dev, int intx) +{ + /* + * We only register one INTx interrupt with the server. map_irq + * callback is required for PCIBus. + */ + return 0; +} + +static void vfu_object_set_irq(void *opaque, int pirq, int level) +{ + vfu_ctx_t *vfu_ctx = opaque; + + if (vfu_ctx && level) { + vfu_irq_trigger(vfu_ctx, 0); + } +} + +static void vfu_object_msi_notify(PCIDevice *pci_dev, unsigned vector) +{ + vfu_ctx_t *vfu_ctx = NULL; + + if (!vfu_object_dev_to_ctx_table) { + return; + } + + vfu_ctx = g_hash_table_lookup(vfu_object_dev_to_ctx_table, pci_dev); + + if (vfu_ctx) { + vfu_irq_trigger(vfu_ctx, vector); + } +} + +static int vfu_object_setup_irqs(VfuObject *o, PCIDevice *pci_dev) +{ + vfu_ctx_t *vfu_ctx = o->vfu_ctx; + int ret; + + ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_INTX_IRQ, 1); + if (ret < 0) { + return ret; + } + + pci_bus_irqs(pci_get_bus(o->pci_dev), vfu_object_set_irq, + vfu_object_map_irq, o->vfu_ctx, 1); + + ret = 0; + if (msix_nr_vectors_allocated(pci_dev)) { + ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_MSIX_IRQ, + msix_nr_vectors_allocated(pci_dev)); + + pci_dev->msix_notify = vfu_object_msi_notify; + } else if (msi_nr_vectors_allocated(pci_dev)) { + ret = vfu_setup_device_nr_irqs(vfu_ctx, VFU_DEV_MSI_IRQ, + msi_nr_vectors_allocated(pci_dev)); + + pci_dev->msi_notify = vfu_object_msi_notify; + } + + if (ret < 0) { + return ret; + } + + return 0; +} + /* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These @@ -478,6 +548,13 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) vfu_object_register_bars(o->vfu_ctx, o->pci_dev); + ret = vfu_object_setup_irqs(o, o->pci_dev); + if (ret < 0) { + error_setg(errp, "vfu: Failed to setup interrupts for %s", + o->device); + goto fail; + } + ret = vfu_realize_ctx(o->vfu_ctx); if (ret < 0) { error_setg(errp, "vfu: Failed to realize device %s- %s", @@ -491,6 +568,8 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + g_hash_table_insert(vfu_object_dev_to_ctx_table, o->pci_dev, o->vfu_ctx); + qemu_set_fd_handler(o->vfu_poll_fd, vfu_object_attach_ctx, NULL, o); return; @@ -552,9 +631,15 @@ static void vfu_object_finalize(Object *obj) o->unplug_blocker = NULL; } + if (o->pci_dev) { + g_hash_table_remove(vfu_object_dev_to_ctx_table, o->pci_dev); + } + o->pci_dev = NULL; if (!k->nr_devs && k->auto_shutdown) { + g_hash_table_destroy(vfu_object_dev_to_ctx_table); + vfu_object_dev_to_ctx_table = NULL; qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN); } @@ -572,6 +657,10 @@ static void vfu_object_class_init(ObjectClass *klass, void *data) k->auto_shutdown = true; + msi_nonbroken = true; + + vfu_object_dev_to_ctx_table = g_hash_table_new_full(NULL, NULL, NULL, NULL); + object_class_property_add(klass, "socket", "SocketAddress", NULL, vfu_object_set_socket, NULL, NULL); object_class_property_set_description(klass, "socket", diff --git a/hw/remote/trace-events b/hw/remote/trace-events index 847d50d88f..c167b3c7a5 100644 --- a/hw/remote/trace-events +++ b/hw/remote/trace-events @@ -12,3 +12,4 @@ vfu_dma_unregister(uint64_t gpa) "vfu: unregistering GPA 0x%"PRIx64"" vfu_bar_register(int i, uint64_t addr, uint64_t size) "vfu: BAR %d: addr 0x%"PRIx64" size 0x%"PRIx64"" vfu_bar_rw_enter(const char *op, uint64_t addr) "vfu: %s request for BAR address 0x%"PRIx64"" vfu_bar_rw_exit(const char *op, uint64_t addr) "vfu: Finished %s of BAR address 0x%"PRIx64"" +vfu_interrupt(int pirq) "vfu: sending interrupt to device - PIRQ %d" -- 2.20.1