Re: [PATCH v5 22/34] hw/vfio: generate new file fd for pseudo device and rebind existing descriptors

[Cédric Le Goater]

On 2/18/26 18:33, Ani Sinha wrote:

On Wed, Feb 18, 2026 at 9:00 PM Cédric Le Goater <c...@redhat.com> wrote:

On 2/18/26 16:07, Ani Sinha wrote:

On 18 Feb 2026, at 7:37 PM, Cédric Le Goater <c...@redhat.com> wrote:

On 2/18/26 12:42, Ani Sinha wrote:

Normally the vfio pseudo device file descriptor lives for the life of the VM.
However, when the kvm VM file descriptor changes, a new file descriptor
for the pseudo device needs to be generated against the new kvm VM descriptor.
Other existing vfio descriptors needs to be reattached to the new pseudo device
descriptor. This change performs the above steps.
Tested-by: Cédric Le Goater <c...@redhat.com>

There is a regression since last version.


'reboot' from the guest and command 'system_reset' from the QEMU
monitor now generate these outputs:

   qemu-system-x86_64: info: virtual machine state has been rebuilt with new 
guest file handle.
   qemu-system-x86_64: info: virtual machine state has been rebuilt with new 
guest file handle.
   qemu-system-x86_64: info: virtual machine state has been rebuilt with new 
guest file handle.
   ...

and QEMU exits after a while.

I have only seen this in SEV-ES with more than one vcpus. Never with TDX or 
SEV-SNP (single or multiple cpus).
On which host/guest type did you see this?

SEV-SNP on a RHEL9 host. Same guest I used before and host says :

    [1816531.409591] kvm_amd: SEV-ES guest requested termination: 0x0:0x0

Strange! I am not sure why KVM thinks it's SEV-ES. I have done all my
SEV-SNP and TDX testing on a fc43 host and for SEV-ES I used a fc42
host. I have not seen this kind of guest termination on SEV-SNP or TDX
on that host. I am sure there are some differences between the RHEL9
host kernel and fc43 kernel.

The same issue occurs with a rhel10 kernel.

C.

Thanks,

C.

Signed-off-by: Ani Sinha <anisi...@redhat.com>

Anyhow this patch looks good.

Reviewed-by: Cédric Le Goater <c...@redhat.com>

Thanks,

C.

---
   hw/vfio/helpers.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++
   1 file changed, 92 insertions(+)
diff --git a/hw/vfio/helpers.c b/hw/vfio/helpers.c
index f68f8165d0..e2bedd15ec 100644
--- a/hw/vfio/helpers.c
+++ b/hw/vfio/helpers.c
@@ -116,6 +116,89 @@ bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info 
*info,
    * we'll re-use it should another vfio device be attached before then.
    */
   int vfio_kvm_device_fd = -1;
+
+/*
+ * Confidential virtual machines:
+ * During reset of confidential vms, the kvm vm file descriptor changes.
+ * In this case, the old vfio kvm file descriptor is
+ * closed and a new descriptor is created against the new kvm vm file
+ * descriptor.
+ */
+
+typedef struct VFIODeviceFd {
+    int fd;
+    QLIST_ENTRY(VFIODeviceFd) node;
+} VFIODeviceFd;
+
+static QLIST_HEAD(, VFIODeviceFd) vfio_device_fds =
+    QLIST_HEAD_INITIALIZER(vfio_device_fds);
+
+static void vfio_device_fd_list_add(int fd)
+{
+    VFIODeviceFd *file_fd;
+    file_fd = g_malloc0(sizeof(*file_fd));
+    file_fd->fd = fd;
+    QLIST_INSERT_HEAD(&vfio_device_fds, file_fd, node);
+}
+
+static void vfio_device_fd_list_remove(int fd)
+{
+    VFIODeviceFd *file_fd, *next;
+
+    QLIST_FOREACH_SAFE(file_fd, &vfio_device_fds, node, next) {
+        if (file_fd->fd == fd) {
+            QLIST_REMOVE(file_fd, node);
+            g_free(file_fd);
+            break;
+        }
+    }
+}
+
+static int vfio_device_fd_rebind(NotifierWithReturn *notifier, void *data,
+                                  Error **errp)
+{
+    VFIODeviceFd *file_fd;
+    int ret = 0;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_FILE,
+        .attr = KVM_DEV_VFIO_FILE_ADD,
+    };
+    struct kvm_create_device cd = {
+        .type = KVM_DEV_TYPE_VFIO,
+    };
+
+    /* we are not interested in pre vmfd change notification */
+    if (((VmfdChangeNotifier *)data)->pre) {
+        return 0;
+    }
+
+    if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
+        error_setg_errno(errp, errno, "Failed to create KVM VFIO device");
+        return -errno;
+    }
+
+    if (vfio_kvm_device_fd != -1) {
+        close(vfio_kvm_device_fd);
+    }
+
+    vfio_kvm_device_fd = cd.fd;
+
+    QLIST_FOREACH(file_fd, &vfio_device_fds, node) {
+        attr.addr = (uint64_t)(unsigned long)&file_fd->fd;
+        if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+            error_setg_errno(errp, errno,
+                             "Failed to add fd %d to KVM VFIO device",
+                             file_fd->fd);
+            ret = -errno;
+        }
+    }
+    return ret;
+}
+
+static struct NotifierWithReturn vfio_vmfd_change_notifier = {
+    .notify = vfio_device_fd_rebind,
+};
+
   #endif
     void vfio_kvm_device_close(void)
@@ -153,6 +236,11 @@ int vfio_kvm_device_add_fd(int fd, Error **errp)
           }
             vfio_kvm_device_fd = cd.fd;
+        /*
+         * If the vm file descriptor changes, add a notifier so that we can
+         * re-create the vfio_kvm_device_fd.
+         */
+        kvm_vmfd_add_change_notifier(&vfio_vmfd_change_notifier);
       }
         if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
@@ -160,6 +248,8 @@ int vfio_kvm_device_add_fd(int fd, Error **errp)
                            fd);
           return -errno;
       }
+
+    vfio_device_fd_list_add(fd);
   #endif
       return 0;
   }
@@ -183,6 +273,8 @@ int vfio_kvm_device_del_fd(int fd, Error **errp)
                            "Failed to remove fd %d from KVM VFIO device", fd);
           return -errno;
       }
+
+    vfio_device_fd_list_remove(fd);
   #endif
       return 0;
   }