Re: [PATCH v2 05/10] hw/vfio/pci: Back the CXL memory with a RAM-device region

[Cédric Le Goater]

On 9/16/26 20:44, mho...@nvidia.com wrote:

From: Manish Honap <mho...@nvidia.com>

mmap the kernel's HDM memory region so its host physical pages back a
RAM-device MemoryRegion. It stays out of the guest address space until
the guest commits its decoder and QEMU learns the GPA to place it at.

AI-used-for: code (prototype)
Signed-off-by: Manish Honap <mho...@nvidia.com>
---
  hw/vfio/pci.c | 34 ++++++++++++++++++++++++++++++++++
  hw/vfio/pci.h |  1 +
  2 files changed, 35 insertions(+)

diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 10b13f200d..4716266595 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -3221,8 +3221,11 @@ bool vfio_pci_populate_device(VFIOPCIDevice *vdev, Error 
**errp)
      return true;
  }

+static void vfio_cxl_teardown(VFIOPCIDevice *vdev);

+
  void vfio_pci_put_device(VFIOPCIDevice *vdev)
  {
+    vfio_cxl_teardown(vdev);
      vfio_display_finalize(vdev);
      vfio_bars_finalize(vdev);
      vfio_cpr_pci_unregister_device(vdev);
@@ -3696,11 +3699,42 @@ static bool vfio_cxl_setup(VFIOPCIDevice *vdev, Error 
**errp)
          return false;
      }

+ /*

+     * The HDM memory is host physical. Set up the region, which installs the
+     * fd read/write path, and mmap it for direct guest access; it is added to
+     * the guest address space only once the guest commits its endpoint 
decoder.
+     */
+    if (vfio_region_setup(OBJECT(vdev), vbasedev, &cxl->mem_region,
+                          cxl->mem_region_index, "cxl-mem", errp)) {
+        return false;
+    }
+    if (vfio_region_mmap(&cxl->mem_region)) {
+        /*
+         * Without mmap the region falls back to the kernel's fd read/write
+         * path, which works but traps every access. Warn rather than fail.
+         */
+        warn_report("vfio-cxl: %s: failed to mmap the HDM memory region; "
+                    "performance may be slow", vbasedev->name);
+    }

vfio_cxl_setup calls vfio_region_setup() then vfio_region_mmap()
back-to-back when other 'normal' BARs split these calls across
vfio_populate_device() and vfio_bar_register(). I guess it is fine
for CXL since it is not a PCI BAR.

      cxl->enabled = true;

return true;

  }

+static void vfio_cxl_teardown(VFIOPCIDevice *vdev)

+{
+    VFIOCXL *cxl = &vdev->cxl;
+
+    if (!cxl->enabled) {
+        return;
+    }
+    if (cxl->mem_region.mem) {
+        vfio_region_exit(&cxl->mem_region);
+        vfio_region_finalize(&cxl->mem_region);

However, the tear down should be split between :

1. vfio_exitfn (unrealize)
   drops mmaps, removes subregions and removes references so the
   MR refcount can reach zero.
2. vfio_pci_finalize (instance_finalize)
   frees the MemoryRegion.

Thanks,

C.

+    }
+}
+
  static void vfio_pci_realize(PCIDevice *pdev, Error **errp)
  {
      ERRP_GUARD();
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index 7fdd695704..06d15e807d 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -134,6 +134,7 @@ typedef struct VFIOCXL {
      uint32_t comp_bar;               /* component BAR carrying that block */
      uint64_t hdm_offset;             /* block offset within the component BAR 
*/
      uint64_t dpa_size;               /* size of the HDM memory region */
+    VFIORegion mem_region;           /* HDM memory, mapped at committed GPA */
  } VFIOCXL;

struct VFIOPCIDevice {