Re: [PATCH for QEMU] hw/vfio: Add VMD Passthrough Quirk
On 4/22/20 10:13 AM, Jon Derrick wrote:
> The VMD endpoint provides a real PCIe domain to the guest, including
> bridges and endpoints. The IOMMU performs Host Physical Address to Guest
> Physical Address translation when assigning downstream endpoint BARs and
> when translating MMIO addresses.
>
> This translation is not desired when assigning bridge windows. When MMIO
> goes to an endpoint after being translated to HPA, the bridge will
> reject the HPA transaction because the bridge window has been programmed
> with translated GPAs.
>
> VMD device 28C0 natively supports passthrough by providing the Host
> Physical Address in shadow registers accessible to the guest for bridge
> window assignment. The shadow registers are valid if bit 1 is set in VMD
> VMLOCK config register 0x70.
>
> This quirk emulates the VMLOCK and HPA shadow registers for all VMD
> device ids which don't natively offer this feature. The Linux VMD driver
> is updated to match the QEMU subsystem id to enable this feature.
>
> Signed-off-by: Jon Derrick
> ---
> hw/vfio/pci-quirks.c | 119 +++
> hw/vfio/pci.c| 7 +++
> hw/vfio/pci.h| 2 +
> hw/vfio/trace-events | 4 ++
> 4 files changed, 132 insertions(+)
>
> diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c
> index 2d348f8237..2fd27cc8f6 100644
> --- a/hw/vfio/pci-quirks.c
> +++ b/hw/vfio/pci-quirks.c
> @@ -1709,3 +1709,122 @@ free_exit:
>
> return ret;
> }
> +
> +/*
> + * The VMD endpoint provides a real PCIe domain to the guest. The IOMMU
> + * performs Host Physical Address to Guest Physical Address translation when
> + * assigning downstream endpoint BARs and when translating MMIO addresses.
> + * However this translation is not desired when assigning bridge windows.
> When
> + * MMIO goes to an endpoint after being translated to HPA, the bridge rejects
> + * the transaction because the window has been programmed with translated
> GPAs.
> + *
> + * VMD uses the Host Physical Address in order to correctly program the
> bridge
> + * windows in its PCIe domain. VMD device 28C0 has HPA shadow registers
> located
> + * at offset 0x2000 in MEMBAR2 (BAR 4). The shadow registers are valid if
> bit 1
> + * is set in the VMD VMLOCK config register 0x70.
> + *
> + * This quirk emulates the VMLOCK and HPA shadow registers for all VMD device
> + * ids which don't natively offer this feature. The subsystem vendor/device
> + * id is set to the QEMU subsystem vendor/device id, where the driver matches
> + * the id to enable this feature.
> + */
> +typedef struct VFIOVMDQuirk {
> +VFIOPCIDevice *vdev;
> +uint64_t membar_phys[2];
> +} VFIOVMDQuirk;
> +
> +static uint64_t vfio_vmd_quirk_read(void *opaque, hwaddr addr, unsigned size)
> +{
> +VFIOVMDQuirk *data = opaque;
> +uint64_t val = 0;
> +
> +memcpy(, (void *)data->membar_phys + addr, size);
> +return val;
> +}
> +
> +static const MemoryRegionOps vfio_vmd_quirk = {
> +.read = vfio_vmd_quirk_read,
> +.endianness = DEVICE_LITTLE_ENDIAN,
> +};
> +
> +#define VMD_VMLOCK 0x70
> +#define VMD_SHADOW 0x2000
> +#define VMD_MEMBAR2 4
> +
> +static int vfio_vmd_emulate_shadow_registers(VFIOPCIDevice *vdev)
> +{
> +VFIOQuirk *quirk;
> +VFIOVMDQuirk *data;
> +PCIDevice *pdev = >pdev;
> +int ret;
> +
> +data = g_malloc0(sizeof(*data));
> +ret = pread(vdev->vbasedev.fd, data->membar_phys, 16,
> +vdev->config_offset + PCI_BASE_ADDRESS_2);
> +if (ret != 16) {
> +error_report("VMD %s cannot read MEMBARs (%d)",
> + vdev->vbasedev.name, ret);
> +g_free(data);
> +return -EFAULT;
> +}
> +
> +quirk = vfio_quirk_alloc(1);
> +quirk->data = data;
> +data->vdev = vdev;
> +
> +/* Emulate Shadow Registers */
> +memory_region_init_io(quirk->mem, OBJECT(vdev), _vmd_quirk, data,
> + "vfio-vmd-quirk", sizeof(data->membar_phys));
> +memory_region_add_subregion_overlap(vdev->bars[VMD_MEMBAR2].region.mem,
> +VMD_SHADOW, quirk->mem, 1);
> +memory_region_set_readonly(quirk->mem, true);
> +memory_region_set_enabled(quirk->mem, true);
> +
> +QLIST_INSERT_HEAD(>bars[VMD_MEMBAR2].quirks, quirk, next);
> +
> +trace_vfio_pci_vmd_quirk_shadow_regs(vdev->vbasedev.name,
> + data->membar_phys[0],
> + data->membar_phys[1]);
> +
> +/* Advertise Shadow Register support */
> +pci_byte_test_and_set_mask(pdev->config + VMD_VMLOCK, 0x2);
> +pci_set_byte(pdev->wmask + VMD_VMLOCK, 0);
> +pci_set_byte(vdev->emulated_config_bits + VMD_VMLOCK, 0x2);
> +
> +trace_vfio_pci_vmd_quirk_vmlock(vdev->vbasedev.name,
> +pci_get_byte(pdev->config + VMD_VMLOCK));
> +
> +/* Drivers can match the subsystem vendor/device id */
> +pci_set_word(pdev->config +
Re: [PATCH for QEMU] hw/vfio: Add VMD Passthrough Quirk
On Wed, 2020-04-22 at 14:36 -0700, no-re...@patchew.org wrote:
> Patchew URL:
> https://patchew.org/QEMU/20200422171305.10923-1-jonathan.derr...@intel.com/
>
>
>
> Hi,
>
> This series failed the asan build test. Please find the testing commands and
> their output below. If you have Docker installed, you can probably reproduce
> it
> locally.
>
> === TEST SCRIPT BEGIN ===
> #!/bin/bash
> export ARCH=x86_64
> make docker-image-fedora V=1 NETWORK=1
> time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1
> === TEST SCRIPT END ===
>
> CC qemu-io-cmds.o
> CC replication.o
> In file included from hw/vfio/trace.c:5:
> /tmp/qemu-test/build/hw/vfio/trace.h:2348:120: error: expected ')'
> qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
> membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
>
> ^
>
It's the extra " at the end, which will be fixed in v2
Thanks for the sanity check
Re: [PATCH for QEMU] hw/vfio: Add VMD Passthrough Quirk
Patchew URL:
https://patchew.org/QEMU/20200422171305.10923-1-jonathan.derr...@intel.com/
Hi,
This series failed the docker-quick@centos7 build test. Please find the testing
commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.
=== TEST SCRIPT BEGIN ===
#!/bin/bash
make docker-image-centos7 V=1 NETWORK=1
time make docker-test-quick@centos7 SHOW_ENV=1 J=14 NETWORK=1
=== TEST SCRIPT END ===
CC stubs/uuid.o
In file included from hw/vfio/trace.c:5:0:
hw/vfio/trace.h: In function '_nocheck__trace_vfio_pci_vmd_quirk_shadow_regs':
hw/vfio/trace.h:2348:9: error: stray '\' in program
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
hw/vfio/trace.h:2348:121: error: expected ')' before 'n'
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
hw/vfio/trace.h:2348:122: error: missing terminating " character [-Werror]
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
hw/vfio/trace.h:2348:121: error: missing terminating " character
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
hw/vfio/trace.h:2351:18: error: format '%d' expects a matching 'int' argument
[-Werror=format=]
, name, mb1, mb2);
^
hw/vfio/trace.h:2351:18: error: format '%zu' expects a matching 'size_t'
argument [-Werror=format=]
hw/vfio/trace.h:2351:18: error: format '%zu' expects a matching 'size_t'
argument [-Werror=format=]
hw/vfio/trace.h:2351:18: error: format '%s' expects a matching 'char *'
argument [-Werror=format=]
hw/vfio/trace.h:2351:18: error: format '%lx' expects a matching 'long unsigned
int' argument [-Werror=format=]
hw/vfio/trace.h:2351:18: error: format '%lx' expects a matching 'long unsigned
int' argument [-Werror=format=]
CC stubs/vm-stop.o
cc1: all warnings being treated as errors
CC stubs/vmstate.o
make: *** [hw/vfio/trace.o] Error 1
make: *** Waiting for unfinished jobs
CC stubs/fd-register.o
Traceback (most recent call last):
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run',
'--label', 'com.qemu.instance.uuid=db534fd3eb71499bb0dc521684cf5cb6', '-u',
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=',
'-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e',
'SHOW_ENV=1', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v',
'/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v',
'/var/tmp/patchew-tester-tmp-uce_gd7q/src/docker-src.2020-04-22-17.37.18.14548:/var/tmp/qemu:z,ro',
'qemu:centos7', '/var/tmp/qemu/run', 'test-quick']' returned non-zero exit
status 2.
filter=--filter=label=com.qemu.instance.uuid=db534fd3eb71499bb0dc521684cf5cb6
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-uce_gd7q/src'
make: *** [docker-run-test-quick@centos7] Error 2
real2m17.142s
user0m8.210s
The full log is available at
http://patchew.org/logs/20200422171305.10923-1-jonathan.derr...@intel.com/testing.docker-quick@centos7/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com
Re: [PATCH for QEMU] hw/vfio: Add VMD Passthrough Quirk
Patchew URL:
https://patchew.org/QEMU/20200422171305.10923-1-jonathan.derr...@intel.com/
Hi,
This series failed the asan build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.
=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-debug@fedora TARGET_LIST=x86_64-softmmu J=14 NETWORK=1
=== TEST SCRIPT END ===
CC qemu-io-cmds.o
CC replication.o
In file included from hw/vfio/trace.c:5:
/tmp/qemu-test/build/hw/vfio/trace.h:2348:120: error: expected ')'
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
/tmp/qemu-test/build/hw/vfio/trace.h:2348:17: note: to match this '('
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
/tmp/qemu-test/build/hw/vfio/trace.h:2348:122: error: missing terminating '"'
character [-Werror,-Winvalid-pp-token]
qemu_log("%d@%zu.%06zu:vfio_pci_vmd_quirk_shadow_regs " "%s
membar1_phys=0x%"PRIx64" membar2_phys=0x%"PRIx64" "\n",
^
CC block/raw-format.o
2 errors generated.
make: *** [/tmp/qemu-test/src/rules.mak:69: hw/vfio/trace.o] Error 1
make: *** Waiting for unfinished jobs
CC block/vmdk.o
Traceback (most recent call last):
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run',
'--label', 'com.qemu.instance.uuid=231ac96de1674d3b887b1d3b8fd4779c', '-u',
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e',
'TARGET_LIST=x86_64-softmmu', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e',
'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache',
'-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v',
'/var/tmp/patchew-tester-tmp-y01wd0hh/src/docker-src.2020-04-22-17.32.38.5605:/var/tmp/qemu:z,ro',
'qemu:fedora', '/var/tmp/qemu/run', 'test-debug']' returned non-zero exit
status 2.
filter=--filter=label=com.qemu.instance.uuid=231ac96de1674d3b887b1d3b8fd4779c
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-y01wd0hh/src'
make: *** [docker-run-test-debug@fedora] Error 2
real3m31.279s
user0m8.228s
The full log is available at
http://patchew.org/logs/20200422171305.10923-1-jonathan.derr...@intel.com/testing.asan/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-de...@redhat.com
[PATCH for QEMU] hw/vfio: Add VMD Passthrough Quirk
The VMD endpoint provides a real PCIe domain to the guest, including
bridges and endpoints. The IOMMU performs Host Physical Address to Guest
Physical Address translation when assigning downstream endpoint BARs and
when translating MMIO addresses.
This translation is not desired when assigning bridge windows. When MMIO
goes to an endpoint after being translated to HPA, the bridge will
reject the HPA transaction because the bridge window has been programmed
with translated GPAs.
VMD device 28C0 natively supports passthrough by providing the Host
Physical Address in shadow registers accessible to the guest for bridge
window assignment. The shadow registers are valid if bit 1 is set in VMD
VMLOCK config register 0x70.
This quirk emulates the VMLOCK and HPA shadow registers for all VMD
device ids which don't natively offer this feature. The Linux VMD driver
is updated to match the QEMU subsystem id to enable this feature.
Signed-off-by: Jon Derrick
---
hw/vfio/pci-quirks.c | 119 +++
hw/vfio/pci.c| 7 +++
hw/vfio/pci.h| 2 +
hw/vfio/trace-events | 4 ++
4 files changed, 132 insertions(+)
diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c
index 2d348f8237..2fd27cc8f6 100644
--- a/hw/vfio/pci-quirks.c
+++ b/hw/vfio/pci-quirks.c
@@ -1709,3 +1709,122 @@ free_exit:
return ret;
}
+
+/*
+ * The VMD endpoint provides a real PCIe domain to the guest. The IOMMU
+ * performs Host Physical Address to Guest Physical Address translation when
+ * assigning downstream endpoint BARs and when translating MMIO addresses.
+ * However this translation is not desired when assigning bridge windows. When
+ * MMIO goes to an endpoint after being translated to HPA, the bridge rejects
+ * the transaction because the window has been programmed with translated GPAs.
+ *
+ * VMD uses the Host Physical Address in order to correctly program the bridge
+ * windows in its PCIe domain. VMD device 28C0 has HPA shadow registers located
+ * at offset 0x2000 in MEMBAR2 (BAR 4). The shadow registers are valid if bit 1
+ * is set in the VMD VMLOCK config register 0x70.
+ *
+ * This quirk emulates the VMLOCK and HPA shadow registers for all VMD device
+ * ids which don't natively offer this feature. The subsystem vendor/device
+ * id is set to the QEMU subsystem vendor/device id, where the driver matches
+ * the id to enable this feature.
+ */
+typedef struct VFIOVMDQuirk {
+VFIOPCIDevice *vdev;
+uint64_t membar_phys[2];
+} VFIOVMDQuirk;
+
+static uint64_t vfio_vmd_quirk_read(void *opaque, hwaddr addr, unsigned size)
+{
+VFIOVMDQuirk *data = opaque;
+uint64_t val = 0;
+
+memcpy(, (void *)data->membar_phys + addr, size);
+return val;
+}
+
+static const MemoryRegionOps vfio_vmd_quirk = {
+.read = vfio_vmd_quirk_read,
+.endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+#define VMD_VMLOCK 0x70
+#define VMD_SHADOW 0x2000
+#define VMD_MEMBAR2 4
+
+static int vfio_vmd_emulate_shadow_registers(VFIOPCIDevice *vdev)
+{
+VFIOQuirk *quirk;
+VFIOVMDQuirk *data;
+PCIDevice *pdev = >pdev;
+int ret;
+
+data = g_malloc0(sizeof(*data));
+ret = pread(vdev->vbasedev.fd, data->membar_phys, 16,
+vdev->config_offset + PCI_BASE_ADDRESS_2);
+if (ret != 16) {
+error_report("VMD %s cannot read MEMBARs (%d)",
+ vdev->vbasedev.name, ret);
+g_free(data);
+return -EFAULT;
+}
+
+quirk = vfio_quirk_alloc(1);
+quirk->data = data;
+data->vdev = vdev;
+
+/* Emulate Shadow Registers */
+memory_region_init_io(quirk->mem, OBJECT(vdev), _vmd_quirk, data,
+ "vfio-vmd-quirk", sizeof(data->membar_phys));
+memory_region_add_subregion_overlap(vdev->bars[VMD_MEMBAR2].region.mem,
+VMD_SHADOW, quirk->mem, 1);
+memory_region_set_readonly(quirk->mem, true);
+memory_region_set_enabled(quirk->mem, true);
+
+QLIST_INSERT_HEAD(>bars[VMD_MEMBAR2].quirks, quirk, next);
+
+trace_vfio_pci_vmd_quirk_shadow_regs(vdev->vbasedev.name,
+ data->membar_phys[0],
+ data->membar_phys[1]);
+
+/* Advertise Shadow Register support */
+pci_byte_test_and_set_mask(pdev->config + VMD_VMLOCK, 0x2);
+pci_set_byte(pdev->wmask + VMD_VMLOCK, 0);
+pci_set_byte(vdev->emulated_config_bits + VMD_VMLOCK, 0x2);
+
+trace_vfio_pci_vmd_quirk_vmlock(vdev->vbasedev.name,
+pci_get_byte(pdev->config + VMD_VMLOCK));
+
+/* Drivers can match the subsystem vendor/device id */
+pci_set_word(pdev->config + PCI_SUBSYSTEM_VENDOR_ID,
+ PCI_SUBVENDOR_ID_REDHAT_QUMRANET);
+pci_set_word(vdev->emulated_config_bits + PCI_SUBSYSTEM_VENDOR_ID, ~0);
+
+pci_set_word(pdev->config + PCI_SUBSYSTEM_ID, PCI_SUBDEVICE_ID_QEMU);
+pci_set_word(vdev->emulated_config_bits + PCI_SUBSYSTEM_ID, ~0);
+
+
