[PULL v1 12/12] docs/system/i386: xenpvh: Add a basic description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 MAINTAINERS |  1 +
 docs/system/i386/xenpvh.rst | 49 +
 docs/system/target-i386.rst |  1 +
 3 files changed, 51 insertions(+)
 create mode 100644 docs/system/i386/xenpvh.rst

diff --git a/MAINTAINERS b/MAINTAINERS
index c2fb0c2f42..c14ac014e2 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -560,6 +560,7 @@ F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
 F: docs/system/arm/xenpvh.rst
+F: docs/system/i386/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
diff --git a/docs/system/i386/xenpvh.rst b/docs/system/i386/xenpvh.rst
new file mode 100644
index 00..354250f073
--- /dev/null
+++ b/docs/system/i386/xenpvh.rst
@@ -0,0 +1,49 @@
+Xen PVH machine (``xenpvh``)
+=
+
+Xen supports a spectrum of types of guests that vary in how they depend
+on HW virtualization features, emulation models and paravirtualization.
+PVH is a mode that uses HW virtualization features (like HVM) but tries
+to avoid emulation models and instead use passthrough or
+paravirtualized devices.
+
+QEMU can be used to provide PV virtio devices on an emulated PCIe controller.
+That is the purpose of this minimal machine.
+
+Supported devices
+-
+
+The x86 Xen PVH QEMU machine provide the following devices:
+
+- RAM
+- GPEX host bridge
+- virtio-pci devices
+
+The idea is to only connect virtio-pci devices but in theory any compatible
+PCI device model will work depending on Xen and guest support.
+
+Running
+---
+
+The Xen tools will typically construct a command-line and launch QEMU
+for you when needed. But here's an example of what it can look like in
+case you need to construct one manually:
+
+.. code-block:: console
+
+qemu-system-i386 -xen-domid 3 -no-shutdown\
+  -chardev 
socket,id=libxl-cmd,path=/var/run/xen/qmp-libxl-3,server=on,wait=off \
+  -mon chardev=libxl-cmd,mode=control \
+  -chardev 
socket,id=libxenstat-cmd,path=/var/run/xen/qmp-libxenstat-3,server=on,wait=off \
+  -mon chardev=libxenstat-cmd,mode=control\
+  -nodefaults \
+  -no-user-config \
+  -xen-attach -name g0\
+  -vnc none   \
+  -display none   \
+  -device virtio-net-pci,id=nic0,netdev=net0,mac=00:16:3e:5c:81:78 \
+  -netdev 
type=tap,id=net0,ifname=vif3.0-emu,br=xenbr0,script=no,downscript=no \
+  -smp 4,maxcpus=4\
+  -nographic  \
+  -machine 
xenpvh,ram-low-base=0,ram-low-size=2147483648,ram-high-base=4294967296,ram-high-size=2147483648,pci-ecam-base=824633720832,pci-ecam-size=268435456,pci-mmio-base=4026531840,pci-mmio-size=33554432,pci-mmio-high-base=824902156288,pci-mmio-high-size=68719476736
 \
+  -m 4096
diff --git a/docs/system/target-i386.rst b/docs/system/target-i386.rst
index 1b8a1f248a..23e84e3ba7 100644
--- a/docs/system/target-i386.rst
+++ b/docs/system/target-i386.rst
@@ -26,6 +26,7 @@ Architectural features
i386/cpu
i386/hyperv
i386/xen
+   i386/xenpvh
i386/kvm-pv
i386/sgx
i386/amd-memory-encryption
-- 
2.43.0

[PULL v1 03/12] hw/arm: xenpvh: Tweak machine description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Tweak machine description to better express that this is
a Xen PVH machine for ARM.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 766a194fa1..5f75cc3779 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -216,7 +216,7 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 {
 
 MachineClass *mc = MACHINE_CLASS(oc);
-mc->desc = "Xen Para-virtualized PC";
+mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
 mc->max_cpus = 1;
 mc->default_machine_opts = "accel=xen";
-- 
2.43.0

[PULL v1 06/12] hw/arm: xenpvh: Move stubbed functions to xen-stubs.c

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/meson.build |  5 -
 hw/arm/xen-stubs.c | 32 
 hw/arm/xen_arm.c   | 20 
 3 files changed, 36 insertions(+), 21 deletions(-)
 create mode 100644 hw/arm/xen-stubs.c

diff --git a/hw/arm/meson.build b/hw/arm/meson.build
index 0c07ab522f..074612b40c 100644
--- a/hw/arm/meson.build
+++ b/hw/arm/meson.build
@@ -59,7 +59,10 @@ arm_ss.add(when: 'CONFIG_FSL_IMX7', if_true: 
files('fsl-imx7.c', 'mcimx7d-sabre.
 arm_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmuv3.c'))
 arm_ss.add(when: 'CONFIG_FSL_IMX6UL', if_true: files('fsl-imx6ul.c', 
'mcimx6ul-evk.c'))
 arm_ss.add(when: 'CONFIG_NRF51_SOC', if_true: files('nrf51_soc.c'))
-arm_ss.add(when: 'CONFIG_XEN', if_true: files('xen_arm.c'))
+arm_ss.add(when: 'CONFIG_XEN', if_true: files(
+  'xen-stubs.c',
+  'xen_arm.c',
+))
 
 system_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmu-common.c'))
 system_ss.add(when: 'CONFIG_CHEETAH', if_true: files('palm.c'))
diff --git a/hw/arm/xen-stubs.c b/hw/arm/xen-stubs.c
new file mode 100644
index 00..4ac6a56a96
--- /dev/null
+++ b/hw/arm/xen-stubs.c
@@ -0,0 +1,32 @@
+/*
+ * Stubs for unimplemented Xen functions for ARM.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "qapi/qapi-commands-migration.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/xen-hvm-common.h"
+#include "hw/xen/arch_hvm.h"
+
+void arch_handle_ioreq(XenIOState *state, ioreq_t *req)
+{
+hw_error("Invalid ioreq type 0x%x\n", req->type);
+return;
+}
+
+void arch_xen_set_memory(XenIOState *state, MemoryRegionSection *section,
+ bool add)
+{
+}
+
+void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+}
+
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+}
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 16b3f00992..f0868e7be5 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -115,26 +115,6 @@ static void xen_init_ram(MachineState *machine)
 memory_region_add_subregion(sysmem, XEN_GRANT_ADDR_OFF, &xen_grants);
 }
 
-void arch_handle_ioreq(XenIOState *state, ioreq_t *req)
-{
-hw_error("Invalid ioreq type 0x%x\n", req->type);
-
-return;
-}
-
-void arch_xen_set_memory(XenIOState *state, MemoryRegionSection *section,
- bool add)
-{
-}
-
-void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t length)
-{
-}
-
-void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
-{
-}
-
 #ifdef CONFIG_TPM
 static void xen_enable_tpm(XenArmState *xam)
 {
-- 
2.43.0

[PULL v1 04/12] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add SMP support for Xen PVH ARM guests.
Create ms->smp.max_cpus ioreq servers to handle hotplug.

Note that ms->smp.max_cpus will be passed to us by the
user (Xen tools) set to the guests maxvcpus.

The value in mc->max_cpus is an absolute maximum for the
-smp option and won't be used to setup ioreq servers unless
the user explicitly specifies it with -smp.

If the user doesn't pass -smp on the command-line, smp.cpus
and smp.max_cpus will default to 1.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 23 +--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 5f75cc3779..fda65d0d8d 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -173,7 +173,7 @@ static void xen_arm_init(MachineState *machine)
 
 xen_init_ram(machine);
 
-xen_register_ioreq(xam->state, machine->smp.cpus, &xen_memory_listener);
+xen_register_ioreq(xam->state, machine->smp.max_cpus, 
&xen_memory_listener);
 
 xen_create_virtio_mmio_devices(xam);
 
@@ -218,7 +218,26 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 MachineClass *mc = MACHINE_CLASS(oc);
 mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
-mc->max_cpus = 1;
+
+/*
+ * mc->max_cpus holds the MAX value allowed in the -smp command-line opts.
+ *
+ * 1. If users don't pass any -smp option:
+ *   ms->smp.cpus will default to 1.
+ *   ms->smp.max_cpus will default to 1.
+ *
+ * 2. If users pass -smp X:
+ *   ms->smp.cpus will be set to X.
+ *   ms->smp.max_cpus will also be set to X.
+ *
+ * 3. If users pass -smp X,maxcpus=Y:
+ *   ms->smp.cpus will be set to X.
+ *   ms->smp.max_cpus will be set to Y.
+ *
+ * In scenarios 2 and 3, if X or Y are set to something larger than
+ * mc->max_cpus, QEMU will bail out with an error message.
+ */
+mc->max_cpus = GUEST_MAX_VCPUS;
 mc->default_machine_opts = "accel=xen";
 /* Set explicitly here to make sure that real ram_size is passed */
 mc->default_ram_size = 0;
-- 
2.43.0

[PULL v1 01/12] MAINTAINERS: Add docs/system/arm/xenpvh.rst

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 3584d6a6c6..c2fb0c2f42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -559,6 +559,7 @@ F: include/hw/xen/
 F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
+F: docs/system/arm/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
-- 
2.43.0

[PULL v1 08/12] hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Rename xen_arm.c -> xen-pvh.c to better express that this
is a PVH machine and to align with x86 HVM and future PVH
machine filenames:
hw/i386/xen/xen-hvm.c
hw/i386/xen/xen-pvh.c (in preparation)

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/meson.build  | 2 +-
 hw/arm/{xen_arm.c => xen-pvh.c} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hw/arm/{xen_arm.c => xen-pvh.c} (100%)

diff --git a/hw/arm/meson.build b/hw/arm/meson.build
index 074612b40c..4059d0be2e 100644
--- a/hw/arm/meson.build
+++ b/hw/arm/meson.build
@@ -61,7 +61,7 @@ arm_ss.add(when: 'CONFIG_FSL_IMX6UL', if_true: 
files('fsl-imx6ul.c', 'mcimx6ul-e
 arm_ss.add(when: 'CONFIG_NRF51_SOC', if_true: files('nrf51_soc.c'))
 arm_ss.add(when: 'CONFIG_XEN', if_true: files(
   'xen-stubs.c',
-  'xen_arm.c',
+  'xen-pvh.c',
 ))
 
 system_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmu-common.c'))
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen-pvh.c
similarity index 100%
rename from hw/arm/xen_arm.c
rename to hw/arm/xen-pvh.c
-- 
2.43.0

[PULL v1 10/12] hw/xen: pvh-common: Add support for creating PCIe/GPEX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add support for optionally creating a PCIe/GPEX controller.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-pvh-common.c | 76 +
 include/hw/xen/xen-pvh-common.h | 29 +
 2 files changed, 105 insertions(+)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 295f920442..28d7168446 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -122,6 +122,64 @@ static void xen_enable_tpm(XenPVHMachineState *s)
 }
 #endif
 
+/*
+ * We use the GPEX PCIe controller with its internal INTX PCI interrupt
+ * swizzling. This swizzling is emulated in QEMU and routes all INTX
+ * interrupts from endpoints down to only 4 INTX interrupts.
+ * See include/hw/pci/pci.h : pci_swizzle()
+ */
+static inline void xenpvh_gpex_init(XenPVHMachineState *s,
+XenPVHMachineClass *xpc,
+MemoryRegion *sysmem)
+{
+MemoryRegion *ecam_reg;
+MemoryRegion *mmio_reg;
+DeviceState *dev;
+int i;
+
+object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
+TYPE_GPEX_HOST);
+dev = DEVICE(&s->pci.gpex);
+sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
+
+ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
+memory_region_add_subregion(sysmem, s->cfg.pci_ecam.base, ecam_reg);
+
+mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
+
+if (s->cfg.pci_mmio.size) {
+memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), "pcie-mmio",
+ mmio_reg,
+ s->cfg.pci_mmio.base, s->cfg.pci_mmio.size);
+memory_region_add_subregion(sysmem, s->cfg.pci_mmio.base,
+&s->pci.mmio_alias);
+}
+
+if (s->cfg.pci_mmio_high.size) {
+memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
+"pcie-mmio-high",
+mmio_reg, s->cfg.pci_mmio_high.base, 
s->cfg.pci_mmio_high.size);
+memory_region_add_subregion(sysmem, s->cfg.pci_mmio_high.base,
+&s->pci.mmio_high_alias);
+}
+
+/*
+ * PVH implementations with PCI enabled must provide set_pci_intx_irq()
+ * and optionally an implementation of set_pci_link_route().
+ */
+assert(xpc->set_pci_intx_irq);
+
+for (i = 0; i < GPEX_NUM_IRQS; i++) {
+qemu_irq irq = qemu_allocate_irq(xpc->set_pci_intx_irq, s, i);
+
+sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
+gpex_set_irq_num(GPEX_HOST(dev), i, s->cfg.pci_intx_irq_base + i);
+if (xpc->set_pci_link_route) {
+xpc->set_pci_link_route(i, s->cfg.pci_intx_irq_base + i);
+}
+}
+}
+
 static void xen_pvh_init(MachineState *ms)
 {
 XenPVHMachineState *s = XEN_PVH_MACHINE(ms);
@@ -152,6 +210,15 @@ static void xen_pvh_init(MachineState *ms)
 }
 #endif
 
+/* Non-zero pci-ecam-size enables PCI.  */
+if (s->cfg.pci_ecam.size) {
+if (s->cfg.pci_ecam.size != 256 * MiB) {
+error_report("pci-ecam-size only supports values 0 or 0x1000");
+exit(EXIT_FAILURE);
+}
+xenpvh_gpex_init(s, xpc, sysmem);
+}
+
 /* Call the implementation specific init.  */
 if (xpc->init) {
 xpc->init(ms);
@@ -200,6 +267,9 @@ XEN_PVH_PROP_MEMMAP(ram_high)
 /* TPM only has a base-addr option.  */
 XEN_PVH_PROP_MEMMAP_BASE(tpm)
 XEN_PVH_PROP_MEMMAP(virtio_mmio)
+XEN_PVH_PROP_MEMMAP(pci_ecam)
+XEN_PVH_PROP_MEMMAP(pci_mmio)
+XEN_PVH_PROP_MEMMAP(pci_mmio_high)
 
 void xen_pvh_class_setup_common_props(XenPVHMachineClass *xpc)
 {
@@ -242,6 +312,12 @@ do {   
   \
 OC_MEMMAP_PROP(oc, "virtio-mmio", virtio_mmio);
 }
 
+if (xpc->has_pci) {
+OC_MEMMAP_PROP(oc, "pci-ecam", pci_ecam);
+OC_MEMMAP_PROP(oc, "pci-mmio", pci_mmio);
+OC_MEMMAP_PROP(oc, "pci-mmio-high", pci_mmio_high);
+}
+
 #ifdef CONFIG_TPM
 if (xpc->has_tpm) {
 object_class_property_add(oc, "tpm-base-addr", "uint64_t",
diff --git a/include/hw/xen/xen-pvh-common.h b/include/hw/xen/xen-pvh-common.h
index 77fd98b9fe..bc09eea936 100644
--- a/include/hw/xen/xen-pvh-common.h
+++ b/include/hw/xen/xen-pvh-common.h
@@ -25,10 +25,29 @@ struct XenPVHMachineClass {
 /* PVH implementation specific init.  */
 void (*init)(MachineState *state);
 
+/*
+ * set_pci_intx_irq - Deliver INTX irqs to the guest.
+ *
+ * @opaque: pointer to XenPVHMachineState.
+ * @irq: IRQ after swizzling, between 0-3.
+ * @level: IRQ level.
+ */
+void (*set_pci_intx_irq)(vo

[PULL v1 00/12] Xen queue

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

The following changes since commit e638d685ec2a0700fb9529cbd1b2823ac4120c53:

  Open 9.2 development tree (2024-09-03 09:18:43 -0700)

are available in the Git repository at:

  https://gitlab.com/edgar.iglesias/qemu.git 
tags/edgar/xen-queue-2024-09-04.for-upstream

for you to fetch changes up to 0b57c8160a2a6c833cfb1d958f08385c4391ab70:

  docs/system/i386: xenpvh: Add a basic description (2024-09-04 16:50:43 +0200)


Edgars Xen queue.

--------
Edgar E. Iglesias (12):
  MAINTAINERS: Add docs/system/arm/xenpvh.rst
  hw/arm: xenpvh: Update file header to use SPDX
  hw/arm: xenpvh: Tweak machine description
  hw/arm: xenpvh: Add support for SMP guests
  hw/arm: xenpvh: Remove double-negation in warning
  hw/arm: xenpvh: Move stubbed functions to xen-stubs.c
  hw/arm: xenpvh: Break out a common PVH machine
  hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c
  hw/arm: xenpvh: Reverse virtio-mmio creation order
  hw/xen: pvh-common: Add support for creating PCIe/GPEX
  hw/i386/xen: Add a Xen PVH x86 machine
  docs/system/i386: xenpvh: Add a basic description

 MAINTAINERS |   2 +
 docs/system/i386/xenpvh.rst |  49 ++
 docs/system/target-i386.rst |   1 +
 hw/arm/meson.build  |   5 +-
 hw/arm/trace-events |   5 -
 hw/arm/xen-pvh.c|  89 ++
 hw/arm/xen-stubs.c  |  32 
 hw/arm/xen_arm.c| 267 -
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 121 ++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 362 
 include/hw/xen/xen-pvh-common.h |  88 ++
 14 files changed, 754 insertions(+), 273 deletions(-)
 create mode 100644 docs/system/i386/xenpvh.rst
 create mode 100644 hw/arm/xen-pvh.c
 create mode 100644 hw/arm/xen-stubs.c
 delete mode 100644 hw/arm/xen_arm.c
 create mode 100644 hw/i386/xen/xen-pvh.c
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

-- 
2.43.0

[PULL v1 11/12] hw/i386/xen: Add a Xen PVH x86 machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a Xen PVH x86 machine based on the abstract PVH Machine.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 121 
 2 files changed, 122 insertions(+)
 create mode 100644 hw/i386/xen/xen-pvh.c

diff --git a/hw/i386/xen/meson.build b/hw/i386/xen/meson.build
index 3f0df8bc07..c73c62b8e3 100644
--- a/hw/i386/xen/meson.build
+++ b/hw/i386/xen/meson.build
@@ -4,6 +4,7 @@ i386_ss.add(when: 'CONFIG_XEN', if_true: files(
 ))
 i386_ss.add(when: ['CONFIG_XEN', xen], if_true: files(
   'xen-hvm.c',
+  'xen-pvh.c',
 ))
 
 i386_ss.add(when: 'CONFIG_XEN_BUS', if_true: files(
diff --git a/hw/i386/xen/xen-pvh.c b/hw/i386/xen/xen-pvh.c
new file mode 100644
index 00..45645667e9
--- /dev/null
+++ b/hw/i386/xen/xen-pvh.c
@@ -0,0 +1,121 @@
+/*
+ * QEMU Xen PVH x86 Machine
+ *
+ * Copyright (c) 2024 Advanced Micro Devices, Inc.
+ * Written by Edgar E. Iglesias 
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/arch_hvm.h"
+#include 
+#include "hw/xen/xen-pvh-common.h"
+
+#define TYPE_XEN_PVH_X86  MACHINE_TYPE_NAME("xenpvh")
+OBJECT_DECLARE_SIMPLE_TYPE(XenPVHx86State, XEN_PVH_X86)
+
+struct XenPVHx86State {
+/*< private >*/
+XenPVHMachineState parent;
+
+DeviceState **cpu;
+};
+
+static DeviceState *xen_pvh_cpu_new(MachineState *ms,
+int64_t apic_id)
+{
+Object *cpu = object_new(ms->cpu_type);
+
+object_property_add_child(OBJECT(ms), "cpu[*]", cpu);
+object_property_set_uint(cpu, "apic-id", apic_id, &error_fatal);
+qdev_realize(DEVICE(cpu), NULL, &error_fatal);
+object_unref(cpu);
+
+return DEVICE(cpu);
+}
+
+static void xen_pvh_init(MachineState *ms)
+{
+XenPVHx86State *xp = XEN_PVH_X86(ms);
+int i;
+
+/* Create dummy cores. This will indirectly create the APIC MSI window.  */
+xp->cpu = g_malloc(sizeof xp->cpu[0] * ms->smp.max_cpus);
+for (i = 0; i < ms->smp.max_cpus; i++) {
+xp->cpu[i] = xen_pvh_cpu_new(ms, i);
+}
+}
+
+static void xen_pvh_instance_init(Object *obj)
+{
+XenPVHMachineState *s = XEN_PVH_MACHINE(obj);
+
+/* Default values.  */
+s->cfg.ram_low = (MemMapEntry) { 0x0, 0x8000U };
+s->cfg.ram_high = (MemMapEntry) { 0xC0ULL, 0x40ULL };
+s->cfg.pci_intx_irq_base = 16;
+}
+
+/*
+ * Deliver INTX interrupts to Xen guest.
+ */
+static void xen_pvh_set_pci_intx_irq(void *opaque, int irq, int level)
+{
+/*
+ * Since QEMU emulates all of the swizziling
+ * We don't want Xen to do any additional swizzling in
+ * xen_set_pci_intx_level() so we always set device to 0.
+ */
+if (xen_set_pci_intx_level(xen_domid, 0, 0, 0, irq, level)) {
+error_report("xendevicemodel_set_pci_intx_level failed");
+}
+}
+
+static void xen_pvh_machine_class_init(ObjectClass *oc, void *data)
+{
+XenPVHMachineClass *xpc = XEN_PVH_MACHINE_CLASS(oc);
+MachineClass *mc = MACHINE_CLASS(oc);
+
+mc->desc = "Xen PVH x86 machine";
+mc->default_cpu_type = TARGET_DEFAULT_CPU_TYPE;
+
+/* mc->max_cpus holds the MAX value allowed in the -smp cmd-line opts. */
+mc->max_cpus = HVM_MAX_VCPUS;
+
+/* We have an implementation specific init to create CPU objects.  */
+xpc->init = xen_pvh_init;
+
+/*
+ * PCI INTX routing.
+ *
+ * We describe the mapping between the 4 INTX interrupt and GSIs
+ * using xen_set_pci_link_route(). xen_pvh_set_pci_intx_irq is
+ * used to deliver the interrupt.
+ */
+xpc->set_pci_intx_irq = xen_pvh_set_pci_intx_irq;
+xpc->set_pci_link_route = xen_set_pci_link_route;
+
+/* List of supported features known to work on PVH x86.  */
+xpc->has_pci = true;
+
+xen_pvh_class_setup_common_props(xpc);
+}
+
+static const TypeInfo xen_pvh_x86_machine_type = {
+.name = TYPE_XEN_PVH_X86,
+.parent = TYPE_XEN_PVH_MACHINE,
+.class_init = xen_pvh_machine_class_init,
+.instance_init = xen_pvh_instance_init,
+.instance_size = sizeof(XenPVHx86State),
+};
+
+static void xen_pvh_machine_register_types(void)
+{
+type_register_static(&xen_pvh_x86_machine_type);
+}
+
+type_init(xen_pvh_machine_register_types)
-- 
2.43.0

[PULL v1 09/12] hw/arm: xenpvh: Reverse virtio-mmio creation order

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

We've been creating the virtio-mmio devices in forwards order
but since the qbus lists prepend (rather than append) entries,
the virtio busses end up with decreasing base address order.

Xen enables virtio-mmio nodes in forwards order so there's been
a missmatch. So far, we've been working around this with an
out-of-tree patch to Xen.

This reverses the order making sure the virtio busses end up
ordered with increasing base addresses avoiding the need to
patch Xen.

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 hw/xen/xen-pvh-common.c | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 880e8143d7..295f920442 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -75,7 +75,18 @@ static void 
xen_create_virtio_mmio_devices(XenPVHMachineState *s)
 {
 int i;
 
-for (i = 0; i < s->cfg.virtio_mmio_num; i++) {
+/*
+ * We create the transports in reverse order. Since qbus_realize()
+ * prepends (not appends) new child buses, the decrementing loop below will
+ * create a list of virtio-mmio buses with increasing base addresses.
+ *
+ * When a -device option is processed from the command line,
+ * qbus_find_recursive() picks the next free virtio-mmio bus in forwards
+ * order.
+ *
+ * This is what the Xen tools expect.
+ */
+for (i = s->cfg.virtio_mmio_num - 1; i >= 0; i--) {
 hwaddr base = s->cfg.virtio_mmio.base + i * s->cfg.virtio_mmio.size;
 qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
  s->cfg.virtio_mmio_irq_base + i);
-- 
2.43.0

[PULL v1 02/12] hw/arm: xenpvh: Update file header to use SPDX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Update file header to use SPDX and remove stray empty
comment line.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 19 +--
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 6fad829ede..766a194fa1 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -1,24 +1,7 @@
 /*
  * QEMU ARM Xen PVH Machine
  *
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to 
deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  */
 
 #include "qemu/osdep.h"
-- 
2.43.0

[PULL v1 05/12] hw/arm: xenpvh: Remove double-negation in warning

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index fda65d0d8d..16b3f00992 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -165,7 +165,7 @@ static void xen_arm_init(MachineState *machine)
 xam->state =  g_new0(XenIOState, 1);
 
 if (machine->ram_size == 0) {
-warn_report("%s non-zero ram size not specified. QEMU machine started"
+warn_report("%s: ram size not specified. QEMU machine started"
 " without IOREQ (no emulated devices including virtio)",
 MACHINE_CLASS(object_get_class(OBJECT(machine)))->desc);
 return;
-- 
2.43.0

[PULL v1 07/12] hw/arm: xenpvh: Break out a common PVH machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Break out a common Xen PVH machine in preparation for
adding a x86 Xen PVH machine.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/trace-events |   5 -
 hw/arm/xen_arm.c| 198 +++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 275 
 include/hw/xen/xen-pvh-common.h |  59 +++
 6 files changed, 358 insertions(+), 184 deletions(-)
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

diff --git a/hw/arm/trace-events b/hw/arm/trace-events
index be6c8f720b..c64ad344bd 100644
--- a/hw/arm/trace-events
+++ b/hw/arm/trace-events
@@ -68,10 +68,5 @@ z2_aer915_send_too_long(int8_t msg) "message too long (%i 
bytes)"
 z2_aer915_send(uint8_t reg, uint8_t value) "reg %d value 0x%02x"
 z2_aer915_event(int8_t event, int8_t len) "i2c event =0x%x len=%d bytes"
 
-# xen_arm.c
-xen_create_virtio_mmio_devices(int i, int irq, uint64_t base) "Created 
virtio-mmio device %d: irq %d base 0x%"PRIx64
-xen_init_ram(uint64_t machine_ram_size) "Initialized xen ram with size 
0x%"PRIx64
-xen_enable_tpm(uint64_t addr) "Connected tpmdev at address 0x%"PRIx64
-
 # bcm2838.c
 bcm2838_gic_set_irq(int irq, int level) "gic irq:%d lvl:%d"
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index f0868e7be5..04cb9855af 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -7,44 +7,12 @@
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "qapi/qapi-commands-migration.h"
-#include "qapi/visitor.h"
 #include "hw/boards.h"
-#include "hw/irq.h"
-#include "hw/sysbus.h"
-#include "sysemu/block-backend.h"
-#include "sysemu/tpm_backend.h"
 #include "sysemu/sysemu.h"
-#include "hw/xen/xen-hvm-common.h"
-#include "sysemu/tpm.h"
+#include "hw/xen/xen-pvh-common.h"
 #include "hw/xen/arch_hvm.h"
-#include "trace.h"
 
 #define TYPE_XEN_ARM  MACHINE_TYPE_NAME("xenpvh")
-OBJECT_DECLARE_SIMPLE_TYPE(XenArmState, XEN_ARM)
-
-static const MemoryListener xen_memory_listener = {
-.region_add = xen_region_add,
-.region_del = xen_region_del,
-.log_start = NULL,
-.log_stop = NULL,
-.log_sync = NULL,
-.log_global_start = NULL,
-.log_global_stop = NULL,
-.priority = MEMORY_LISTENER_PRIORITY_ACCEL,
-};
-
-struct XenArmState {
-/*< private >*/
-MachineState parent;
-
-XenIOState *state;
-
-struct {
-uint64_t tpm_base_addr;
-} cfg;
-};
-
-static MemoryRegion ram_lo, ram_hi;
 
 /*
  * VIRTIO_MMIO_DEV_SIZE is imported from tools/libs/light/libxl_arm.c under Xen
@@ -57,147 +25,26 @@ static MemoryRegion ram_lo, ram_hi;
 #define NR_VIRTIO_MMIO_DEVICES   \
(GUEST_VIRTIO_MMIO_SPI_LAST - GUEST_VIRTIO_MMIO_SPI_FIRST)
 
-static void xen_set_irq(void *opaque, int irq, int level)
-{
-if (xendevicemodel_set_irq_level(xen_dmod, xen_domid, irq, level)) {
-error_report("xendevicemodel_set_irq_level failed");
-}
-}
-
-static void xen_create_virtio_mmio_devices(XenArmState *xam)
-{
-int i;
-
-for (i = 0; i < NR_VIRTIO_MMIO_DEVICES; i++) {
-hwaddr base = GUEST_VIRTIO_MMIO_BASE + i * VIRTIO_MMIO_DEV_SIZE;
-qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i);
-
-sysbus_create_simple("virtio-mmio", base, irq);
-
-trace_xen_create_virtio_mmio_devices(i,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i,
- base);
-}
-}
-
-static void xen_init_ram(MachineState *machine)
+static void xen_arm_instance_init(Object *obj)
 {
-MemoryRegion *sysmem = get_system_memory();
-ram_addr_t block_len, ram_size[GUEST_RAM_BANKS];
-
-trace_xen_init_ram(machine->ram_size);
-if (machine->ram_size <= GUEST_RAM0_SIZE) {
-ram_size[0] = machine->ram_size;
-ram_size[1] = 0;
-block_len = GUEST_RAM0_BASE + ram_size[0];
-} else {
-ram_size[0] = GUEST_RAM0_SIZE;
-ram_size[1] = machine->ram_size - GUEST_RAM0_SIZE;
-block_len = GUEST_RAM1_BASE + ram_size[1];
-}
+XenPVHMachineState *s = XEN_PVH_MACHINE(obj);
 
-memory_region_init_ram(&xen_memory, NULL, "xen.ram", block_len,
-   &error_fatal);
+/* Default values.  */
+s->cfg.ram_low = (MemMapEntry) { GUEST_RAM0_BASE, GUEST_RAM0_SIZE };
+s->cfg.ram_high = (MemMapEntry) { GUEST_RAM1_BASE, GUEST_RAM1_SIZE };
 
-memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo", &xen_memory,
- GU

[PATCH v2 10/12] hw/xen: pvh-common: Add support for creating PCIe/GPEX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add support for optionally creating a PCIe/GPEX controller.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-pvh-common.c | 76 +
 include/hw/xen/xen-pvh-common.h | 29 +
 2 files changed, 105 insertions(+)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 295f920442..28d7168446 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -122,6 +122,64 @@ static void xen_enable_tpm(XenPVHMachineState *s)
 }
 #endif
 
+/*
+ * We use the GPEX PCIe controller with its internal INTX PCI interrupt
+ * swizzling. This swizzling is emulated in QEMU and routes all INTX
+ * interrupts from endpoints down to only 4 INTX interrupts.
+ * See include/hw/pci/pci.h : pci_swizzle()
+ */
+static inline void xenpvh_gpex_init(XenPVHMachineState *s,
+XenPVHMachineClass *xpc,
+MemoryRegion *sysmem)
+{
+MemoryRegion *ecam_reg;
+MemoryRegion *mmio_reg;
+DeviceState *dev;
+int i;
+
+object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
+TYPE_GPEX_HOST);
+dev = DEVICE(&s->pci.gpex);
+sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
+
+ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
+memory_region_add_subregion(sysmem, s->cfg.pci_ecam.base, ecam_reg);
+
+mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
+
+if (s->cfg.pci_mmio.size) {
+memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), "pcie-mmio",
+ mmio_reg,
+ s->cfg.pci_mmio.base, s->cfg.pci_mmio.size);
+memory_region_add_subregion(sysmem, s->cfg.pci_mmio.base,
+&s->pci.mmio_alias);
+}
+
+if (s->cfg.pci_mmio_high.size) {
+memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
+"pcie-mmio-high",
+mmio_reg, s->cfg.pci_mmio_high.base, 
s->cfg.pci_mmio_high.size);
+memory_region_add_subregion(sysmem, s->cfg.pci_mmio_high.base,
+&s->pci.mmio_high_alias);
+}
+
+/*
+ * PVH implementations with PCI enabled must provide set_pci_intx_irq()
+ * and optionally an implementation of set_pci_link_route().
+ */
+assert(xpc->set_pci_intx_irq);
+
+for (i = 0; i < GPEX_NUM_IRQS; i++) {
+qemu_irq irq = qemu_allocate_irq(xpc->set_pci_intx_irq, s, i);
+
+sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
+gpex_set_irq_num(GPEX_HOST(dev), i, s->cfg.pci_intx_irq_base + i);
+if (xpc->set_pci_link_route) {
+xpc->set_pci_link_route(i, s->cfg.pci_intx_irq_base + i);
+}
+}
+}
+
 static void xen_pvh_init(MachineState *ms)
 {
 XenPVHMachineState *s = XEN_PVH_MACHINE(ms);
@@ -152,6 +210,15 @@ static void xen_pvh_init(MachineState *ms)
 }
 #endif
 
+/* Non-zero pci-ecam-size enables PCI.  */
+if (s->cfg.pci_ecam.size) {
+if (s->cfg.pci_ecam.size != 256 * MiB) {
+error_report("pci-ecam-size only supports values 0 or 0x1000");
+exit(EXIT_FAILURE);
+}
+xenpvh_gpex_init(s, xpc, sysmem);
+}
+
 /* Call the implementation specific init.  */
 if (xpc->init) {
 xpc->init(ms);
@@ -200,6 +267,9 @@ XEN_PVH_PROP_MEMMAP(ram_high)
 /* TPM only has a base-addr option.  */
 XEN_PVH_PROP_MEMMAP_BASE(tpm)
 XEN_PVH_PROP_MEMMAP(virtio_mmio)
+XEN_PVH_PROP_MEMMAP(pci_ecam)
+XEN_PVH_PROP_MEMMAP(pci_mmio)
+XEN_PVH_PROP_MEMMAP(pci_mmio_high)
 
 void xen_pvh_class_setup_common_props(XenPVHMachineClass *xpc)
 {
@@ -242,6 +312,12 @@ do {   
   \
 OC_MEMMAP_PROP(oc, "virtio-mmio", virtio_mmio);
 }
 
+if (xpc->has_pci) {
+OC_MEMMAP_PROP(oc, "pci-ecam", pci_ecam);
+OC_MEMMAP_PROP(oc, "pci-mmio", pci_mmio);
+OC_MEMMAP_PROP(oc, "pci-mmio-high", pci_mmio_high);
+}
+
 #ifdef CONFIG_TPM
 if (xpc->has_tpm) {
 object_class_property_add(oc, "tpm-base-addr", "uint64_t",
diff --git a/include/hw/xen/xen-pvh-common.h b/include/hw/xen/xen-pvh-common.h
index 77fd98b9fe..bc09eea936 100644
--- a/include/hw/xen/xen-pvh-common.h
+++ b/include/hw/xen/xen-pvh-common.h
@@ -25,10 +25,29 @@ struct XenPVHMachineClass {
 /* PVH implementation specific init.  */
 void (*init)(MachineState *state);
 
+/*
+ * set_pci_intx_irq - Deliver INTX irqs to the guest.
+ *
+ * @opaque: pointer to XenPVHMachineState.
+ * @irq: IRQ after swizzling, between 0-3.
+ * @level: IRQ level.
+ */
+void (*set_pci_intx_irq)(void *opaque, int irq, int level);
+
+ 

[PATCH v2 12/12] docs/system/i386: xenpvh: Add a basic description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 MAINTAINERS |  1 +
 docs/system/i386/xenpvh.rst | 49 +
 docs/system/target-i386.rst |  1 +
 3 files changed, 51 insertions(+)
 create mode 100644 docs/system/i386/xenpvh.rst

diff --git a/MAINTAINERS b/MAINTAINERS
index c2fb0c2f42..c14ac014e2 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -560,6 +560,7 @@ F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
 F: docs/system/arm/xenpvh.rst
+F: docs/system/i386/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
diff --git a/docs/system/i386/xenpvh.rst b/docs/system/i386/xenpvh.rst
new file mode 100644
index 00..354250f073
--- /dev/null
+++ b/docs/system/i386/xenpvh.rst
@@ -0,0 +1,49 @@
+Xen PVH machine (``xenpvh``)
+=
+
+Xen supports a spectrum of types of guests that vary in how they depend
+on HW virtualization features, emulation models and paravirtualization.
+PVH is a mode that uses HW virtualization features (like HVM) but tries
+to avoid emulation models and instead use passthrough or
+paravirtualized devices.
+
+QEMU can be used to provide PV virtio devices on an emulated PCIe controller.
+That is the purpose of this minimal machine.
+
+Supported devices
+-
+
+The x86 Xen PVH QEMU machine provide the following devices:
+
+- RAM
+- GPEX host bridge
+- virtio-pci devices
+
+The idea is to only connect virtio-pci devices but in theory any compatible
+PCI device model will work depending on Xen and guest support.
+
+Running
+---
+
+The Xen tools will typically construct a command-line and launch QEMU
+for you when needed. But here's an example of what it can look like in
+case you need to construct one manually:
+
+.. code-block:: console
+
+qemu-system-i386 -xen-domid 3 -no-shutdown\
+  -chardev 
socket,id=libxl-cmd,path=/var/run/xen/qmp-libxl-3,server=on,wait=off \
+  -mon chardev=libxl-cmd,mode=control \
+  -chardev 
socket,id=libxenstat-cmd,path=/var/run/xen/qmp-libxenstat-3,server=on,wait=off \
+  -mon chardev=libxenstat-cmd,mode=control\
+  -nodefaults \
+  -no-user-config \
+  -xen-attach -name g0\
+  -vnc none   \
+  -display none   \
+  -device virtio-net-pci,id=nic0,netdev=net0,mac=00:16:3e:5c:81:78 \
+  -netdev 
type=tap,id=net0,ifname=vif3.0-emu,br=xenbr0,script=no,downscript=no \
+  -smp 4,maxcpus=4\
+  -nographic  \
+  -machine 
xenpvh,ram-low-base=0,ram-low-size=2147483648,ram-high-base=4294967296,ram-high-size=2147483648,pci-ecam-base=824633720832,pci-ecam-size=268435456,pci-mmio-base=4026531840,pci-mmio-size=33554432,pci-mmio-high-base=824902156288,pci-mmio-high-size=68719476736
 \
+  -m 4096
diff --git a/docs/system/target-i386.rst b/docs/system/target-i386.rst
index 1b8a1f248a..23e84e3ba7 100644
--- a/docs/system/target-i386.rst
+++ b/docs/system/target-i386.rst
@@ -26,6 +26,7 @@ Architectural features
i386/cpu
i386/hyperv
i386/xen
+   i386/xenpvh
i386/kvm-pv
i386/sgx
i386/amd-memory-encryption
-- 
2.43.0

[PATCH v2 08/12] hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Rename xen_arm.c -> xen-pvh.c to better express that this
is a PVH machine and to align with x86 HVM and future PVH
machine filenames:
hw/i386/xen/xen-hvm.c
hw/i386/xen/xen-pvh.c (in preparation)

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/meson.build  | 2 +-
 hw/arm/{xen_arm.c => xen-pvh.c} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hw/arm/{xen_arm.c => xen-pvh.c} (100%)

diff --git a/hw/arm/meson.build b/hw/arm/meson.build
index 074612b40c..4059d0be2e 100644
--- a/hw/arm/meson.build
+++ b/hw/arm/meson.build
@@ -61,7 +61,7 @@ arm_ss.add(when: 'CONFIG_FSL_IMX6UL', if_true: 
files('fsl-imx6ul.c', 'mcimx6ul-e
 arm_ss.add(when: 'CONFIG_NRF51_SOC', if_true: files('nrf51_soc.c'))
 arm_ss.add(when: 'CONFIG_XEN', if_true: files(
   'xen-stubs.c',
-  'xen_arm.c',
+  'xen-pvh.c',
 ))
 
 system_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmu-common.c'))
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen-pvh.c
similarity index 100%
rename from hw/arm/xen_arm.c
rename to hw/arm/xen-pvh.c
-- 
2.43.0

[PATCH v2 06/12] hw/arm: xenpvh: Move stubbed functions to xen-stubs.c

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/meson.build |  5 -
 hw/arm/xen-stubs.c | 32 
 hw/arm/xen_arm.c   | 20 
 3 files changed, 36 insertions(+), 21 deletions(-)
 create mode 100644 hw/arm/xen-stubs.c

diff --git a/hw/arm/meson.build b/hw/arm/meson.build
index 0c07ab522f..074612b40c 100644
--- a/hw/arm/meson.build
+++ b/hw/arm/meson.build
@@ -59,7 +59,10 @@ arm_ss.add(when: 'CONFIG_FSL_IMX7', if_true: 
files('fsl-imx7.c', 'mcimx7d-sabre.
 arm_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmuv3.c'))
 arm_ss.add(when: 'CONFIG_FSL_IMX6UL', if_true: files('fsl-imx6ul.c', 
'mcimx6ul-evk.c'))
 arm_ss.add(when: 'CONFIG_NRF51_SOC', if_true: files('nrf51_soc.c'))
-arm_ss.add(when: 'CONFIG_XEN', if_true: files('xen_arm.c'))
+arm_ss.add(when: 'CONFIG_XEN', if_true: files(
+  'xen-stubs.c',
+  'xen_arm.c',
+))
 
 system_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmu-common.c'))
 system_ss.add(when: 'CONFIG_CHEETAH', if_true: files('palm.c'))
diff --git a/hw/arm/xen-stubs.c b/hw/arm/xen-stubs.c
new file mode 100644
index 00..4ac6a56a96
--- /dev/null
+++ b/hw/arm/xen-stubs.c
@@ -0,0 +1,32 @@
+/*
+ * Stubs for unimplemented Xen functions for ARM.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "qapi/qapi-commands-migration.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/xen-hvm-common.h"
+#include "hw/xen/arch_hvm.h"
+
+void arch_handle_ioreq(XenIOState *state, ioreq_t *req)
+{
+hw_error("Invalid ioreq type 0x%x\n", req->type);
+return;
+}
+
+void arch_xen_set_memory(XenIOState *state, MemoryRegionSection *section,
+ bool add)
+{
+}
+
+void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+}
+
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+}
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 16b3f00992..f0868e7be5 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -115,26 +115,6 @@ static void xen_init_ram(MachineState *machine)
 memory_region_add_subregion(sysmem, XEN_GRANT_ADDR_OFF, &xen_grants);
 }
 
-void arch_handle_ioreq(XenIOState *state, ioreq_t *req)
-{
-hw_error("Invalid ioreq type 0x%x\n", req->type);
-
-return;
-}
-
-void arch_xen_set_memory(XenIOState *state, MemoryRegionSection *section,
- bool add)
-{
-}
-
-void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t length)
-{
-}
-
-void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
-{
-}
-
 #ifdef CONFIG_TPM
 static void xen_enable_tpm(XenArmState *xam)
 {
-- 
2.43.0

[PATCH v2 04/12] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add SMP support for Xen PVH ARM guests.
Create ms->smp.max_cpus ioreq servers to handle hotplug.

Note that ms->smp.max_cpus will be passed to us by the
user (Xen tools) set to the guests maxvcpus.

The value in mc->max_cpus is an absolute maximum for the
-smp option and won't be used to setup ioreq servers unless
the user explicitly specifies it with -smp.

If the user doesn't pass -smp on the command-line, smp.cpus
and smp.max_cpus will default to 1.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/xen_arm.c | 23 +--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 5f75cc3779..fda65d0d8d 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -173,7 +173,7 @@ static void xen_arm_init(MachineState *machine)
 
 xen_init_ram(machine);
 
-xen_register_ioreq(xam->state, machine->smp.cpus, &xen_memory_listener);
+xen_register_ioreq(xam->state, machine->smp.max_cpus, 
&xen_memory_listener);
 
 xen_create_virtio_mmio_devices(xam);
 
@@ -218,7 +218,26 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 MachineClass *mc = MACHINE_CLASS(oc);
 mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
-mc->max_cpus = 1;
+
+/*
+ * mc->max_cpus holds the MAX value allowed in the -smp command-line opts.
+ *
+ * 1. If users don't pass any -smp option:
+ *   ms->smp.cpus will default to 1.
+ *   ms->smp.max_cpus will default to 1.
+ *
+ * 2. If users pass -smp X:
+ *   ms->smp.cpus will be set to X.
+ *   ms->smp.max_cpus will also be set to X.
+ *
+ * 3. If users pass -smp X,maxcpus=Y:
+ *   ms->smp.cpus will be set to X.
+ *   ms->smp.max_cpus will be set to Y.
+ *
+ * In scenarios 2 and 3, if X or Y are set to something larger than
+ * mc->max_cpus, QEMU will bail out with an error message.
+ */
+mc->max_cpus = GUEST_MAX_VCPUS;
 mc->default_machine_opts = "accel=xen";
 /* Set explicitly here to make sure that real ram_size is passed */
 mc->default_ram_size = 0;
-- 
2.43.0

[PATCH v2 11/12] hw/i386/xen: Add a Xen PVH x86 machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a Xen PVH x86 machine based on the abstract PVH Machine.

Signed-off-by: Edgar E. Iglesias 
---
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 121 
 2 files changed, 122 insertions(+)
 create mode 100644 hw/i386/xen/xen-pvh.c

diff --git a/hw/i386/xen/meson.build b/hw/i386/xen/meson.build
index 3f0df8bc07..c73c62b8e3 100644
--- a/hw/i386/xen/meson.build
+++ b/hw/i386/xen/meson.build
@@ -4,6 +4,7 @@ i386_ss.add(when: 'CONFIG_XEN', if_true: files(
 ))
 i386_ss.add(when: ['CONFIG_XEN', xen], if_true: files(
   'xen-hvm.c',
+  'xen-pvh.c',
 ))
 
 i386_ss.add(when: 'CONFIG_XEN_BUS', if_true: files(
diff --git a/hw/i386/xen/xen-pvh.c b/hw/i386/xen/xen-pvh.c
new file mode 100644
index 00..45645667e9
--- /dev/null
+++ b/hw/i386/xen/xen-pvh.c
@@ -0,0 +1,121 @@
+/*
+ * QEMU Xen PVH x86 Machine
+ *
+ * Copyright (c) 2024 Advanced Micro Devices, Inc.
+ * Written by Edgar E. Iglesias 
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/arch_hvm.h"
+#include 
+#include "hw/xen/xen-pvh-common.h"
+
+#define TYPE_XEN_PVH_X86  MACHINE_TYPE_NAME("xenpvh")
+OBJECT_DECLARE_SIMPLE_TYPE(XenPVHx86State, XEN_PVH_X86)
+
+struct XenPVHx86State {
+/*< private >*/
+XenPVHMachineState parent;
+
+DeviceState **cpu;
+};
+
+static DeviceState *xen_pvh_cpu_new(MachineState *ms,
+int64_t apic_id)
+{
+Object *cpu = object_new(ms->cpu_type);
+
+object_property_add_child(OBJECT(ms), "cpu[*]", cpu);
+object_property_set_uint(cpu, "apic-id", apic_id, &error_fatal);
+qdev_realize(DEVICE(cpu), NULL, &error_fatal);
+object_unref(cpu);
+
+return DEVICE(cpu);
+}
+
+static void xen_pvh_init(MachineState *ms)
+{
+XenPVHx86State *xp = XEN_PVH_X86(ms);
+int i;
+
+/* Create dummy cores. This will indirectly create the APIC MSI window.  */
+xp->cpu = g_malloc(sizeof xp->cpu[0] * ms->smp.max_cpus);
+for (i = 0; i < ms->smp.max_cpus; i++) {
+xp->cpu[i] = xen_pvh_cpu_new(ms, i);
+}
+}
+
+static void xen_pvh_instance_init(Object *obj)
+{
+XenPVHMachineState *s = XEN_PVH_MACHINE(obj);
+
+/* Default values.  */
+s->cfg.ram_low = (MemMapEntry) { 0x0, 0x8000U };
+s->cfg.ram_high = (MemMapEntry) { 0xC0ULL, 0x40ULL };
+s->cfg.pci_intx_irq_base = 16;
+}
+
+/*
+ * Deliver INTX interrupts to Xen guest.
+ */
+static void xen_pvh_set_pci_intx_irq(void *opaque, int irq, int level)
+{
+/*
+ * Since QEMU emulates all of the swizziling
+ * We don't want Xen to do any additional swizzling in
+ * xen_set_pci_intx_level() so we always set device to 0.
+ */
+if (xen_set_pci_intx_level(xen_domid, 0, 0, 0, irq, level)) {
+error_report("xendevicemodel_set_pci_intx_level failed");
+}
+}
+
+static void xen_pvh_machine_class_init(ObjectClass *oc, void *data)
+{
+XenPVHMachineClass *xpc = XEN_PVH_MACHINE_CLASS(oc);
+MachineClass *mc = MACHINE_CLASS(oc);
+
+mc->desc = "Xen PVH x86 machine";
+mc->default_cpu_type = TARGET_DEFAULT_CPU_TYPE;
+
+/* mc->max_cpus holds the MAX value allowed in the -smp cmd-line opts. */
+mc->max_cpus = HVM_MAX_VCPUS;
+
+/* We have an implementation specific init to create CPU objects.  */
+xpc->init = xen_pvh_init;
+
+/*
+ * PCI INTX routing.
+ *
+ * We describe the mapping between the 4 INTX interrupt and GSIs
+ * using xen_set_pci_link_route(). xen_pvh_set_pci_intx_irq is
+ * used to deliver the interrupt.
+ */
+xpc->set_pci_intx_irq = xen_pvh_set_pci_intx_irq;
+xpc->set_pci_link_route = xen_set_pci_link_route;
+
+/* List of supported features known to work on PVH x86.  */
+xpc->has_pci = true;
+
+xen_pvh_class_setup_common_props(xpc);
+}
+
+static const TypeInfo xen_pvh_x86_machine_type = {
+.name = TYPE_XEN_PVH_X86,
+.parent = TYPE_XEN_PVH_MACHINE,
+.class_init = xen_pvh_machine_class_init,
+.instance_init = xen_pvh_instance_init,
+.instance_size = sizeof(XenPVHx86State),
+};
+
+static void xen_pvh_machine_register_types(void)
+{
+type_register_static(&xen_pvh_x86_machine_type);
+}
+
+type_init(xen_pvh_machine_register_types)
-- 
2.43.0

[PATCH v2 09/12] hw/arm: xenpvh: Reverse virtio-mmio creation order

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

We've been creating the virtio-mmio devices in forwards order
but since the qbus lists prepend (rather than append) entries,
the virtio busses end up with decreasing base address order.

Xen enables virtio-mmio nodes in forwards order so there's been
a missmatch. So far, we've been working around this with an
out-of-tree patch to Xen.

This reverses the order making sure the virtio busses end up
ordered with increasing base addresses avoiding the need to
patch Xen.

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 hw/xen/xen-pvh-common.c | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 880e8143d7..295f920442 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -75,7 +75,18 @@ static void 
xen_create_virtio_mmio_devices(XenPVHMachineState *s)
 {
 int i;
 
-for (i = 0; i < s->cfg.virtio_mmio_num; i++) {
+/*
+ * We create the transports in reverse order. Since qbus_realize()
+ * prepends (not appends) new child buses, the decrementing loop below will
+ * create a list of virtio-mmio buses with increasing base addresses.
+ *
+ * When a -device option is processed from the command line,
+ * qbus_find_recursive() picks the next free virtio-mmio bus in forwards
+ * order.
+ *
+ * This is what the Xen tools expect.
+ */
+for (i = s->cfg.virtio_mmio_num - 1; i >= 0; i--) {
 hwaddr base = s->cfg.virtio_mmio.base + i * s->cfg.virtio_mmio.size;
 qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
  s->cfg.virtio_mmio_irq_base + i);
-- 
2.43.0

[PATCH v2 00/12] xen: pvh: Partial QOM:fication with new x86 PVH machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This series breaks out parts of the ARM PVH support into an abstract
machine that other targets can reuse.. There's a bit of refactoring
and some bug-fixes along the way.

Finally we add a new x86 xen-pvh machine.

The corresponding changes in Xen for PVH x86 are work in progress
(by Xenia Ragiadakou). You can find a current version here:
https://github.com/edgarigl/xen/tree/edgar/virtio-pvh-upstream 

I've briefly described the steps to run Xen PVH x86 guests here
(including example guest kernel config, xl.cfg and xl command-lines):
https://github.com/edgarigl/docs/blob/master/xen/pvh/xenpvh-x86.md

Cheers,
Edgar

ChangeLog:

v1 -> v2:
* Use an abstract PVH Machine to share code between arch specific
  PVH machines. This allows more sharing than a using a device component.
* Add comments describing our PCI INTX support.
* Allow the PVH target specific implementation to provide PCI
  INTX delivery via class methods.
* Clarify use of mc->max_cpus in comments.
* Use GUEST_MAX_VCPUS mc->max_cpus on ARM.
* Use HVM_MAX_VCPUS mc->max_cpus on x86.
* pvh-common: Directly use s->cfg. members when creating gpex.
* pvh-common: Clarify use of pci-ecam-size in comments.
* x86/xen-pvh: Create smp.max_cpus nr of dummy cores (not smp.cpus).
* x86/xen-pvh: Replace static array holding dummy cores with a dynamic one.
* Move stubbed Xen/ARM functions from hw/arm/xen_arm.c into xen-stubs.c

Edgar E. Iglesias (12):
  MAINTAINERS: Add docs/system/arm/xenpvh.rst
  hw/arm: xenpvh: Update file header to use SPDX
  hw/arm: xenpvh: Tweak machine description
  hw/arm: xenpvh: Add support for SMP guests
  hw/arm: xenpvh: Remove double-negation in warning
  hw/arm: xenpvh: Move stubbed functions to xen-stubs.c
  hw/arm: xenpvh: Break out a common PVH machine
  hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c
  hw/arm: xenpvh: Reverse virtio-mmio creation order
  hw/xen: pvh-common: Add support for creating PCIe/GPEX
  hw/i386/xen: Add a Xen PVH x86 machine
  docs/system/i386: xenpvh: Add a basic description

 MAINTAINERS |   2 +
 docs/system/i386/xenpvh.rst |  49 +
 docs/system/target-i386.rst |   1 +
 hw/arm/meson.build  |   5 +-
 hw/arm/trace-events |   5 -
 hw/arm/xen-pvh.c|  89 
 hw/arm/xen-stubs.c  |  32 +++
 hw/arm/xen_arm.c| 267 ---
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 121 +++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 362 
 include/hw/xen/xen-pvh-common.h |  88 
 14 files changed, 754 insertions(+), 273 deletions(-)
 create mode 100644 docs/system/i386/xenpvh.rst
 create mode 100644 hw/arm/xen-pvh.c
 create mode 100644 hw/arm/xen-stubs.c
 delete mode 100644 hw/arm/xen_arm.c
 create mode 100644 hw/i386/xen/xen-pvh.c
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

-- 
2.43.0

[PATCH v2 02/12] hw/arm: xenpvh: Update file header to use SPDX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Update file header to use SPDX and remove stray empty
comment line.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 19 +--
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 6fad829ede..766a194fa1 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -1,24 +1,7 @@
 /*
  * QEMU ARM Xen PVH Machine
  *
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to 
deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  */
 
 #include "qemu/osdep.h"
-- 
2.43.0

[PATCH v2 05/12] hw/arm: xenpvh: Remove double-negation in warning

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/xen_arm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index fda65d0d8d..16b3f00992 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -165,7 +165,7 @@ static void xen_arm_init(MachineState *machine)
 xam->state =  g_new0(XenIOState, 1);
 
 if (machine->ram_size == 0) {
-warn_report("%s non-zero ram size not specified. QEMU machine started"
+warn_report("%s: ram size not specified. QEMU machine started"
 " without IOREQ (no emulated devices including virtio)",
 MACHINE_CLASS(object_get_class(OBJECT(machine)))->desc);
 return;
-- 
2.43.0

[PATCH v2 07/12] hw/arm: xenpvh: Break out a common PVH machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Break out a common Xen PVH machine in preparation for
adding a x86 Xen PVH machine.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/trace-events |   5 -
 hw/arm/xen_arm.c| 198 +++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 275 
 include/hw/xen/xen-pvh-common.h |  59 +++
 6 files changed, 358 insertions(+), 184 deletions(-)
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

diff --git a/hw/arm/trace-events b/hw/arm/trace-events
index be6c8f720b..c64ad344bd 100644
--- a/hw/arm/trace-events
+++ b/hw/arm/trace-events
@@ -68,10 +68,5 @@ z2_aer915_send_too_long(int8_t msg) "message too long (%i 
bytes)"
 z2_aer915_send(uint8_t reg, uint8_t value) "reg %d value 0x%02x"
 z2_aer915_event(int8_t event, int8_t len) "i2c event =0x%x len=%d bytes"
 
-# xen_arm.c
-xen_create_virtio_mmio_devices(int i, int irq, uint64_t base) "Created 
virtio-mmio device %d: irq %d base 0x%"PRIx64
-xen_init_ram(uint64_t machine_ram_size) "Initialized xen ram with size 
0x%"PRIx64
-xen_enable_tpm(uint64_t addr) "Connected tpmdev at address 0x%"PRIx64
-
 # bcm2838.c
 bcm2838_gic_set_irq(int irq, int level) "gic irq:%d lvl:%d"
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index f0868e7be5..04cb9855af 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -7,44 +7,12 @@
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "qapi/qapi-commands-migration.h"
-#include "qapi/visitor.h"
 #include "hw/boards.h"
-#include "hw/irq.h"
-#include "hw/sysbus.h"
-#include "sysemu/block-backend.h"
-#include "sysemu/tpm_backend.h"
 #include "sysemu/sysemu.h"
-#include "hw/xen/xen-hvm-common.h"
-#include "sysemu/tpm.h"
+#include "hw/xen/xen-pvh-common.h"
 #include "hw/xen/arch_hvm.h"
-#include "trace.h"
 
 #define TYPE_XEN_ARM  MACHINE_TYPE_NAME("xenpvh")
-OBJECT_DECLARE_SIMPLE_TYPE(XenArmState, XEN_ARM)
-
-static const MemoryListener xen_memory_listener = {
-.region_add = xen_region_add,
-.region_del = xen_region_del,
-.log_start = NULL,
-.log_stop = NULL,
-.log_sync = NULL,
-.log_global_start = NULL,
-.log_global_stop = NULL,
-.priority = MEMORY_LISTENER_PRIORITY_ACCEL,
-};
-
-struct XenArmState {
-/*< private >*/
-MachineState parent;
-
-XenIOState *state;
-
-struct {
-uint64_t tpm_base_addr;
-} cfg;
-};
-
-static MemoryRegion ram_lo, ram_hi;
 
 /*
  * VIRTIO_MMIO_DEV_SIZE is imported from tools/libs/light/libxl_arm.c under Xen
@@ -57,147 +25,26 @@ static MemoryRegion ram_lo, ram_hi;
 #define NR_VIRTIO_MMIO_DEVICES   \
(GUEST_VIRTIO_MMIO_SPI_LAST - GUEST_VIRTIO_MMIO_SPI_FIRST)
 
-static void xen_set_irq(void *opaque, int irq, int level)
-{
-if (xendevicemodel_set_irq_level(xen_dmod, xen_domid, irq, level)) {
-error_report("xendevicemodel_set_irq_level failed");
-}
-}
-
-static void xen_create_virtio_mmio_devices(XenArmState *xam)
-{
-int i;
-
-for (i = 0; i < NR_VIRTIO_MMIO_DEVICES; i++) {
-hwaddr base = GUEST_VIRTIO_MMIO_BASE + i * VIRTIO_MMIO_DEV_SIZE;
-qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i);
-
-sysbus_create_simple("virtio-mmio", base, irq);
-
-trace_xen_create_virtio_mmio_devices(i,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i,
- base);
-}
-}
-
-static void xen_init_ram(MachineState *machine)
+static void xen_arm_instance_init(Object *obj)
 {
-MemoryRegion *sysmem = get_system_memory();
-ram_addr_t block_len, ram_size[GUEST_RAM_BANKS];
-
-trace_xen_init_ram(machine->ram_size);
-if (machine->ram_size <= GUEST_RAM0_SIZE) {
-ram_size[0] = machine->ram_size;
-ram_size[1] = 0;
-block_len = GUEST_RAM0_BASE + ram_size[0];
-} else {
-ram_size[0] = GUEST_RAM0_SIZE;
-ram_size[1] = machine->ram_size - GUEST_RAM0_SIZE;
-block_len = GUEST_RAM1_BASE + ram_size[1];
-}
+XenPVHMachineState *s = XEN_PVH_MACHINE(obj);
 
-memory_region_init_ram(&xen_memory, NULL, "xen.ram", block_len,
-   &error_fatal);
+/* Default values.  */
+s->cfg.ram_low = (MemMapEntry) { GUEST_RAM0_BASE, GUEST_RAM0_SIZE };
+s->cfg.ram_high = (MemMapEntry) { GUEST_RAM1_BASE, GUEST_RAM1_SIZE };
 
-memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo", &xen_memory,
- GUEST_RAM0_BASE, ram_size[0]);
-

[PATCH v2 03/12] hw/arm: xenpvh: Tweak machine description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Tweak machine description to better express that this is
a Xen PVH machine for ARM.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/arm/xen_arm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 766a194fa1..5f75cc3779 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -216,7 +216,7 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 {
 
 MachineClass *mc = MACHINE_CLASS(oc);
-mc->desc = "Xen Para-virtualized PC";
+mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
 mc->max_cpus = 1;
 mc->default_machine_opts = "accel=xen";
-- 
2.43.0

[PATCH v2 01/12] MAINTAINERS: Add docs/system/arm/xenpvh.rst

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
Acked-by: Stefano Stabellini 
---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 3584d6a6c6..c2fb0c2f42 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -559,6 +559,7 @@ F: include/hw/xen/
 F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
+F: docs/system/arm/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
-- 
2.43.0

Re: [PATCH v1 04/10] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

On Sat, Aug 17, 2024 at 2:45 AM Jason Andryuk  wrote:

> On 2024-08-16 12:53, Stefano Stabellini wrote:
> > On Fri, 16 Aug 2024, Edgar E. Iglesias wrote:
> >> On Thu, Aug 15, 2024 at 2:30 AM Stefano Stabellini <
> sstabell...@kernel.org> wrote:
> >>    On Wed, 14 Aug 2024, Edgar E. Iglesias wrote:
> >>> On Tue, Aug 13, 2024 at 03:52:32PM -0700, Stefano Stabellini
> wrote:
> >>> > On Tue, 13 Aug 2024, Edgar E. Iglesias wrote:
> >>> > > On Mon, Aug 12, 2024 at 06:47:17PM -0700, Stefano
> Stabellini wrote:
> >>    > > > > On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> >>> > > > > From: "Edgar E. Iglesias" 
> >>> > > > >
> >>> > > > > Add SMP support for Xen PVH ARM guests. Create
> max_cpus ioreq
> >>> > > > > servers to handle hotplug.
> >>> > > > >
> >>> > > > > Signed-off-by: Edgar E. Iglesias <
> edgar.igles...@amd.com>
> >>> > > > > ---
> >>> > > > >  hw/arm/xen_arm.c | 5 +++--
> >>> > > > >  1 file changed, 3 insertions(+), 2 deletions(-)
> >>> > > > >
> >>> > > > > diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
> >>> > > > > index 5f75cc3779..ef8315969c 100644
> >>> > > > > --- a/hw/arm/xen_arm.c
> >>> > > > > +++ b/hw/arm/xen_arm.c
> >>> > > > > @@ -173,7 +173,7 @@ static void
> xen_arm_init(MachineState *machine)
> >>> > > > >
> >>> > > > >  xen_init_ram(machine);
> >>> > > > >
> >>> > > > > -xen_register_ioreq(xam->state, machine->smp.cpus,
> &xen_memory_listener);
> >>> > > > > +xen_register_ioreq(xam->state,
> machine->smp.max_cpus, &xen_memory_listener);
> >>> > > > >
> >>> > > > >  xen_create_virtio_mmio_devices(xam);
> >>> > > > >
> >>> > > > > @@ -218,7 +218,8 @@ static void
> xen_arm_machine_class_init(ObjectClass *oc, void *data)
> >>> > > > >  MachineClass *mc = MACHINE_CLASS(oc);
> >>> > > > >  mc->desc = "Xen PVH ARM machine";
> >>> > > > >  mc->init = xen_arm_init;
> >>> > > > > -mc->max_cpus = 1;
> >>> > > > > +/* MAX number of vcpus supported by Xen.  */
> >>> > > > > +mc->max_cpus = GUEST_MAX_VCPUS;
> >>> > > >
> >>> > > > Will this cause allocations of data structures with 128
> elements?
> >>> > > > Looking at hw/xen/xen-hvm-common.c:xen_do_ioreq_register
> it seems
> >>> > > > possible? Or
> hw/xen/xen-hvm-common.c:xen_do_ioreq_register is called
> >>> > >
> >>> > > Yes, in theory there's probably overhead with this but as
> you correctly
> >>> > > noted below, a PVH aware xl will set the max_cpus option
> to a lower value.
> >>> > >
> >>> > > With a non-pvh aware xl, I was a little worried about the
> overhead
> >>> > > but I couldn't see any visible slow-down on ARM neither in
> boot or in network
> >>> > > performance (I didn't run very sophisticated benchmarks).
> >>> >
> >>> > What do you mean by "non-pvh aware xl"? All useful versions
> of xl
> >>> > support pvh?
> >>>
> >>>
> >>> I mean an xl without our PVH patches merged.
> >>> xl in upstream doesn't know much about PVH yet.
> >>> Even for ARM, we're still carrying significant patches in our
> tree.
> >>
> >>Oh I see. In that case, I don't think we need to support
> "non-pvh aware xl".
> >>
> >>
> >>> > > > later on with the precise vCPU value which should be
> provided to QEMU
> >>> > > &g

Re: [PATCH v1 04/10] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

On Thu, Aug 15, 2024 at 2:30 AM Stefano Stabellini 
wrote:

> On Wed, 14 Aug 2024, Edgar E. Iglesias wrote:
> > On Tue, Aug 13, 2024 at 03:52:32PM -0700, Stefano Stabellini wrote:
> > > On Tue, 13 Aug 2024, Edgar E. Iglesias wrote:
> > > > On Mon, Aug 12, 2024 at 06:47:17PM -0700, Stefano Stabellini wrote:
> > > > > On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > > > > > From: "Edgar E. Iglesias" 
> > > > > >
> > > > > > Add SMP support for Xen PVH ARM guests. Create max_cpus ioreq
> > > > > > servers to handle hotplug.
> > > > > >
> > > > > > Signed-off-by: Edgar E. Iglesias 
> > > > > > ---
> > > > > >  hw/arm/xen_arm.c | 5 +++--
> > > > > >  1 file changed, 3 insertions(+), 2 deletions(-)
> > > > > >
> > > > > > diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
> > > > > > index 5f75cc3779..ef8315969c 100644
> > > > > > --- a/hw/arm/xen_arm.c
> > > > > > +++ b/hw/arm/xen_arm.c
> > > > > > @@ -173,7 +173,7 @@ static void xen_arm_init(MachineState
> *machine)
> > > > > >
> > > > > >  xen_init_ram(machine);
> > > > > >
> > > > > > -xen_register_ioreq(xam->state, machine->smp.cpus,
> &xen_memory_listener);
> > > > > > +xen_register_ioreq(xam->state, machine->smp.max_cpus,
> &xen_memory_listener);
> > > > > >
> > > > > >  xen_create_virtio_mmio_devices(xam);
> > > > > >
> > > > > > @@ -218,7 +218,8 @@ static void
> xen_arm_machine_class_init(ObjectClass *oc, void *data)
> > > > > >  MachineClass *mc = MACHINE_CLASS(oc);
> > > > > >  mc->desc = "Xen PVH ARM machine";
> > > > > >  mc->init = xen_arm_init;
> > > > > > -mc->max_cpus = 1;
> > > > > > +/* MAX number of vcpus supported by Xen.  */
> > > > > > +mc->max_cpus = GUEST_MAX_VCPUS;
> > > > >
> > > > > Will this cause allocations of data structures with 128 elements?
> > > > > Looking at hw/xen/xen-hvm-common.c:xen_do_ioreq_register it seems
> > > > > possible? Or hw/xen/xen-hvm-common.c:xen_do_ioreq_register is
> called
> > > >
> > > > Yes, in theory there's probably overhead with this but as you
> correctly
> > > > noted below, a PVH aware xl will set the max_cpus option to a lower
> value.
> > > >
> > > > With a non-pvh aware xl, I was a little worried about the overhead
> > > > but I couldn't see any visible slow-down on ARM neither in boot or
> in network
> > > > performance (I didn't run very sophisticated benchmarks).
> > >
> > > What do you mean by "non-pvh aware xl"? All useful versions of xl
> > > support pvh?
> >
> >
> > I mean an xl without our PVH patches merged.
> > xl in upstream doesn't know much about PVH yet.
> > Even for ARM, we're still carrying significant patches in our tree.
>
> Oh I see. In that case, I don't think we need to support "non-pvh aware
> xl".
>
>
> > > > > later on with the precise vCPU value which should be provided to
> QEMU
> > > > > via the -smp command line option
> > > > > (tools/libs/light/libxl_dm.c:libxl__build_device_model_args_new)?
> > > >
> > > > Yes, a pvh aware xl will for example pass -smp 2,maxcpus=4 based on
> > > > values from the xl.cfg. If the user doesn't set maxvcpus in xl.cfg,
> xl
> > > > will set maxvcpus to the same value as vcpus.
> > >
> > > OK good. In that case if this is just an initial value meant to be
> > > overwritten, I think it is best to keep it as 1.
> >
> > Sorry but that won't work. I think the confusion here may be that
> > it's easy to mix up mc->max_cpus and machine->smp.max_cpus, these are
> > not the same. They have different purposes.
> >
> > I'll try to clarify the 3 values in play.
> >
> > machine-smp.cpus:
> > Number of guest vcpus active at boot.
> > Passed to QEMU via the -smp command-line option.
> > We don't use this value in QEMU's ARM PVH machines.
> >
> > machine->smp.max_cpus:
> > Max number of vcpus that the guest can use (equal or larger than
> machine-smp.cpus

Re: [PATCH v1 09/10] hw/i386/xen: Add a Xen PVH x86 machine

[Edgar E. Iglesias]

On Mon, Aug 12, 2024 at 06:48:52PM -0700, Stefano Stabellini wrote:
> On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> > 
> > This adds a Xen PVH x86 machine based on the PVH Common
> > module used by the ARM PVH machine.
> > 
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/i386/xen/meson.build |   1 +
> >  hw/i386/xen/xen-pvh.c   | 196 
> >  2 files changed, 197 insertions(+)
> >  create mode 100644 hw/i386/xen/xen-pvh.c
> > 
> > diff --git a/hw/i386/xen/meson.build b/hw/i386/xen/meson.build
> > index 3f0df8bc07..c73c62b8e3 100644
> > --- a/hw/i386/xen/meson.build
> > +++ b/hw/i386/xen/meson.build
> > @@ -4,6 +4,7 @@ i386_ss.add(when: 'CONFIG_XEN', if_true: files(
> >  ))
> >  i386_ss.add(when: ['CONFIG_XEN', xen], if_true: files(
> >'xen-hvm.c',
> > +  'xen-pvh.c',
> >  ))
> >  
> >  i386_ss.add(when: 'CONFIG_XEN_BUS', if_true: files(
> > diff --git a/hw/i386/xen/xen-pvh.c b/hw/i386/xen/xen-pvh.c
> > new file mode 100644
> > index 000000..9c3d3fc58d
> > --- /dev/null
> > +++ b/hw/i386/xen/xen-pvh.c
> > @@ -0,0 +1,196 @@
> > +/*
> > + * QEMU Xen PVH x86 Machine
> > + *
> > + * Copyright (c) 2024 Advanced Micro Devices, Inc.
> > + * Written by Edgar E. Iglesias 
> > + *
> > + * SPDX-License-Identifier: GPL-2.0-or-later
> > + */
> > +
> > +#include "qemu/osdep.h"
> > +#include "qapi/error.h"
> > +#include "qapi/visitor.h"
> > +#include "qemu/error-report.h"
> > +#include "hw/boards.h"
> > +#include "sysemu/sysemu.h"
> > +#include "hw/xen/arch_hvm.h"
> > +#include "hw/xen/xen.h"
> > +#include "hw/xen/xen-pvh-common.h"
> > +
> > +#define TYPE_XEN_PVH_X86  MACHINE_TYPE_NAME("xenpvh")
> > +OBJECT_DECLARE_SIMPLE_TYPE(XenPVHx86State, XEN_PVH_X86)
> > +
> > +#define PVH_MAX_CPUS 128
> > +
> > +struct XenPVHx86State {
> > +/*< private >*/
> > +MachineState parent;
> > +
> > +DeviceState *cpu[PVH_MAX_CPUS];
> > +XenPVHCommonState pvh;
> > +
> > +/*
> > + * We provide these properties to allow Xen to move things to other
> > + * addresses for example when users need to accomodate the memory-map
> > + * for 1:1 mapped devices/memory.
> > + */
> > +struct {
> > +MemMapEntry ram_low, ram_high;
> > +MemMapEntry pci_ecam, pci_mmio, pci_mmio_high;
> 
> Can we use the same properties already present under XenPVHCommonState?
> 
> 
> 
> > +} cfg;
> > +};
> > +
> > +static void xenpvh_cpu_new(MachineState *ms,
> > +   XenPVHx86State *xp,
> > +   int cpu_idx,
> > +   int64_t apic_id)
> > +{
> > +Object *cpu = object_new(ms->cpu_type);
> > +
> > +object_property_add_child(OBJECT(ms), "cpu[*]", cpu);
> > +object_property_set_uint(cpu, "apic-id", apic_id, &error_fatal);
> > +qdev_realize(DEVICE(cpu), NULL, &error_fatal);
> > +object_unref(cpu);
> > +
> > +xp->cpu[cpu_idx] = DEVICE(cpu);
> 
> 
> Why do we need to create CPU devices in QEMU given that we are only
> doing device emulation? I guess it is because 
> 
> 
> 
> > +}
> > +
> > +static void xenpvh_init(MachineState *ms)
> > +{
> > +XenPVHx86State *xp = XEN_PVH_X86(ms);
> > +const struct {
> > +const char *name;
> > +MemMapEntry *map;
> > +} map[] = {
> > +{ "ram-low", &xp->cfg.ram_low },
> > +{ "ram-high", &xp->cfg.ram_high },
> > +{ "pci-ecam", &xp->cfg.pci_ecam },
> > +{ "pci-mmio", &xp->cfg.pci_mmio },
> > +{ "pci-mmio-high", &xp->cfg.pci_mmio_high },
> > +};
> > +int i;
> > +
> > +object_initialize_child(OBJECT(ms), "pvh", &xp->pvh, 
> > TYPE_XEN_PVH_COMMON);
> > +object_property_set_int(OBJECT(&xp->pvh), "max-cpus", ms->smp.max_cpus,
> > +&error_abort);
> > +object_property_set_int(OBJECT(&xp->pvh), "ram-size", ms->ram_size,
> > +&error_abort);
> > +
> > +   

Re: [PATCH v1 08/10] hw/xen: pvh-common: Add support for creating PCIe/GPEX

[Edgar E. Iglesias]

On Mon, Aug 12, 2024 at 06:48:37PM -0700, Stefano Stabellini wrote:
> On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> > 
> > Add support for optionally creating a PCIe/GPEX controller.
> > 
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/xen/xen-pvh-common.c | 66 +
> >  include/hw/xen/xen-pvh-common.h | 10 -
> >  2 files changed, 75 insertions(+), 1 deletion(-)
> > 
> > diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
> > index 69a2dbdb6d..b1432e4bd9 100644
> > --- a/hw/xen/xen-pvh-common.c
> > +++ b/hw/xen/xen-pvh-common.c
> > @@ -120,6 +120,59 @@ static void xen_enable_tpm(XenPVHCommonState *s)
> >  }
> >  #endif
> >  
> > +static void xen_set_pci_intx_irq(void *opaque, int irq, int level)
> > +{
> > +if (xen_set_pci_intx_level(xen_domid, 0, 0, 0, irq, level)) {
> 
> Looking at the implementation of XEN_DMOP_set_pci_intx_level in
> xen/arch/x86/hvm/dm.c, it looks like the device parameter of
> xen_set_pci_intx_level is required?

Yes, by setting device = 0, we're bypassing the irq swizzling in Xen.
I'll try to clarify below.


> 
> 
> > +error_report("xendevicemodel_set_pci_intx_level failed");
> > +}
> > +}
> > +
> > +static inline void xenpvh_gpex_init(XenPVHCommonState *s,
> > +MemoryRegion *sysmem,
> > +hwaddr ecam_base, hwaddr ecam_size,
> > +hwaddr mmio_base, hwaddr mmio_size,
> > +hwaddr mmio_high_base,
> > +hwaddr mmio_high_size,
> > +int intx_irq_base)
> > +{
> > +MemoryRegion *ecam_reg;
> > +MemoryRegion *mmio_reg;
> > +DeviceState *dev;
> > +int i;
> > +
> > +object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
> > +TYPE_GPEX_HOST);
> > +dev = DEVICE(&s->pci.gpex);
> > +sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
> > +
> > +ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
> > +memory_region_add_subregion(sysmem, ecam_base, ecam_reg);
> 
> I notice we don't use ecam_size anywhere? Is that because the size is
> standard?

Yes. we could remove the size property, having it slightly simplifies the
prop setting code (keeping these memmap prop-pairs alike) but it's not a big 
deal.

> 
> 
> > +mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
> > +
> > +if (mmio_size) {
> > +memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), 
> > "pcie-mmio",
> > + mmio_reg, mmio_base, mmio_size);
> > +memory_region_add_subregion(sysmem, mmio_base, &s->pci.mmio_alias);
> > +}
> > +
> > +if (mmio_high_size) {
> > +memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
> > +"pcie-mmio-high",
> > +mmio_reg, mmio_high_base, mmio_high_size);
> > +memory_region_add_subregion(sysmem, mmio_high_base,
> > +&s->pci.mmio_high_alias);
> > +}
> > +
> > +for (i = 0; i < GPEX_NUM_IRQS; i++) {
> > +qemu_irq irq = qemu_allocate_irq(xen_set_pci_intx_irq, s, i);
> > +
> > +sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
> > +gpex_set_irq_num(GPEX_HOST(dev), i, intx_irq_base + i);
> > +xen_set_pci_link_route(i, intx_irq_base + i);
> 
> xen_set_pci_link_route is not currently implemented on ARM?
> 
> Looking at hw/i386/pc_piix.c:piix_intx_routing_notifier_xen it seems
> that the routing is much more complex over there. But looking at other
> machines that use GPEX such as hw/arm/virt.c it looks like the routing
> is straightforward the same way as in this patch.
> 
> I thought that PCI interrupt pin swizzling was required, but maybe not ?
> 
> It is totally fine if we do something different, simpler, than
> hw/i386/pc_piix.c:piix_intx_routing_notifier_xen. I just want to make
> sure that things remain consistent between ARM and x86, and also between
> Xen and QEMU view of virtual PCI interrupt routing.
>

Good questions. The following is the way I understand things but I may
ofcourse be wrong.

Yes, we're doing things differently than hw/i386/pc_piix.c mainly
because we're using the GPEX PCIe host bridge with it's inte

Re: [PATCH v1 05/10] hw/arm: xenpvh: Break out a common PVH module

[Edgar E. Iglesias]

On Mon, Aug 12, 2024 at 06:47:51PM -0700, Stefano Stabellini wrote:
> On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> > 
> > Break out a common Xen PVH module in preparation for
> > adding a x86 Xen PVH Machine.
> > 
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/arm/trace-events |   5 -
> >  hw/arm/xen_arm.c| 154 ++
> >  hw/xen/meson.build  |   1 +
> >  hw/xen/trace-events |   4 +
> >  hw/xen/xen-pvh-common.c | 185 
> >  include/hw/xen/xen-pvh-common.h |  45 
> >  6 files changed, 269 insertions(+), 125 deletions(-)
> >  create mode 100644 hw/xen/xen-pvh-common.c
> >  create mode 100644 include/hw/xen/xen-pvh-common.h
> > 
> > diff --git a/hw/arm/trace-events b/hw/arm/trace-events
> > index be6c8f720b..c64ad344bd 100644
> > --- a/hw/arm/trace-events
> > +++ b/hw/arm/trace-events
> > @@ -68,10 +68,5 @@ z2_aer915_send_too_long(int8_t msg) "message too long 
> > (%i bytes)"
> >  z2_aer915_send(uint8_t reg, uint8_t value) "reg %d value 0x%02x"
> >  z2_aer915_event(int8_t event, int8_t len) "i2c event =0x%x len=%d bytes"
> >  
> > -# xen_arm.c
> > -xen_create_virtio_mmio_devices(int i, int irq, uint64_t base) "Created 
> > virtio-mmio device %d: irq %d base 0x%"PRIx64
> > -xen_init_ram(uint64_t machine_ram_size) "Initialized xen ram with size 
> > 0x%"PRIx64
> > -xen_enable_tpm(uint64_t addr) "Connected tpmdev at address 0x%"PRIx64
> > -
> >  # bcm2838.c
> >  bcm2838_gic_set_irq(int irq, int level) "gic irq:%d lvl:%d"
> > diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
> > index ef8315969c..b8a5c09bdf 100644
> > --- a/hw/arm/xen_arm.c
> > +++ b/hw/arm/xen_arm.c
> > @@ -12,40 +12,25 @@
> >  #include "hw/irq.h"
> >  #include "hw/sysbus.h"
> >  #include "sysemu/block-backend.h"
> > -#include "sysemu/tpm_backend.h"
> >  #include "sysemu/sysemu.h"
> > -#include "hw/xen/xen-hvm-common.h"
> > +#include "hw/xen/xen-pvh-common.h"
> >  #include "sysemu/tpm.h"
> >  #include "hw/xen/arch_hvm.h"
> > -#include "trace.h"
> >  
> >  #define TYPE_XEN_ARM  MACHINE_TYPE_NAME("xenpvh")
> >  OBJECT_DECLARE_SIMPLE_TYPE(XenArmState, XEN_ARM)
> >  
> > -static const MemoryListener xen_memory_listener = {
> > -.region_add = xen_region_add,
> > -.region_del = xen_region_del,
> > -.log_start = NULL,
> > -.log_stop = NULL,
> > -.log_sync = NULL,
> > -.log_global_start = NULL,
> > -.log_global_stop = NULL,
> > -.priority = MEMORY_LISTENER_PRIORITY_ACCEL,
> > -};
> > -
> >  struct XenArmState {
> >  /*< private >*/
> >  MachineState parent;
> >  
> > -XenIOState *state;
> > +XenPVHCommonState pvh;
> >  
> >  struct {
> >  uint64_t tpm_base_addr;
> >  } cfg;
> >  };
> >  
> > -static MemoryRegion ram_lo, ram_hi;
> > -
> >  /*
> >   * VIRTIO_MMIO_DEV_SIZE is imported from tools/libs/light/libxl_arm.c 
> > under Xen
> >   * repository.
> > @@ -57,64 +42,6 @@ static MemoryRegion ram_lo, ram_hi;
> >  #define NR_VIRTIO_MMIO_DEVICES   \
> > (GUEST_VIRTIO_MMIO_SPI_LAST - GUEST_VIRTIO_MMIO_SPI_FIRST)
> >  
> > -static void xen_set_irq(void *opaque, int irq, int level)
> > -{
> > -if (xendevicemodel_set_irq_level(xen_dmod, xen_domid, irq, level)) {
> > -error_report("xendevicemodel_set_irq_level failed");
> > -}
> > -}
> > -
> > -static void xen_create_virtio_mmio_devices(XenArmState *xam)
> > -{
> > -int i;
> > -
> > -for (i = 0; i < NR_VIRTIO_MMIO_DEVICES; i++) {
> > -hwaddr base = GUEST_VIRTIO_MMIO_BASE + i * VIRTIO_MMIO_DEV_SIZE;
> > -qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
> > - GUEST_VIRTIO_MMIO_SPI_FIRST + i);
> > -
> > -sysbus_create_simple("virtio-mmio", base, irq);
> > -
> > -trace_xen_create_virtio_mmio_devices(i,
> > - GUEST_VIRTIO_MMIO_SPI_FIRST + 
> > i,
> > - base);
> > -}
> > -}
> 
> There are 4 trivial functions in this file:
> 

Re: [PATCH v1 04/10] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

On Tue, Aug 13, 2024 at 03:52:32PM -0700, Stefano Stabellini wrote:
> On Tue, 13 Aug 2024, Edgar E. Iglesias wrote:
> > On Mon, Aug 12, 2024 at 06:47:17PM -0700, Stefano Stabellini wrote:
> > > On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > > > From: "Edgar E. Iglesias" 
> > > > 
> > > > Add SMP support for Xen PVH ARM guests. Create max_cpus ioreq
> > > > servers to handle hotplug.
> > > > 
> > > > Signed-off-by: Edgar E. Iglesias 
> > > > ---
> > > >  hw/arm/xen_arm.c | 5 +++--
> > > >  1 file changed, 3 insertions(+), 2 deletions(-)
> > > > 
> > > > diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
> > > > index 5f75cc3779..ef8315969c 100644
> > > > --- a/hw/arm/xen_arm.c
> > > > +++ b/hw/arm/xen_arm.c
> > > > @@ -173,7 +173,7 @@ static void xen_arm_init(MachineState *machine)
> > > >  
> > > >  xen_init_ram(machine);
> > > >  
> > > > -xen_register_ioreq(xam->state, machine->smp.cpus, 
> > > > &xen_memory_listener);
> > > > +xen_register_ioreq(xam->state, machine->smp.max_cpus, 
> > > > &xen_memory_listener);
> > > >  
> > > >  xen_create_virtio_mmio_devices(xam);
> > > >  
> > > > @@ -218,7 +218,8 @@ static void xen_arm_machine_class_init(ObjectClass 
> > > > *oc, void *data)
> > > >  MachineClass *mc = MACHINE_CLASS(oc);
> > > >  mc->desc = "Xen PVH ARM machine";
> > > >  mc->init = xen_arm_init;
> > > > -mc->max_cpus = 1;
> > > > +/* MAX number of vcpus supported by Xen.  */
> > > > +mc->max_cpus = GUEST_MAX_VCPUS;
> > > 
> > > Will this cause allocations of data structures with 128 elements?
> > > Looking at hw/xen/xen-hvm-common.c:xen_do_ioreq_register it seems
> > > possible? Or hw/xen/xen-hvm-common.c:xen_do_ioreq_register is called
> > 
> > Yes, in theory there's probably overhead with this but as you correctly
> > noted below, a PVH aware xl will set the max_cpus option to a lower value.
> > 
> > With a non-pvh aware xl, I was a little worried about the overhead
> > but I couldn't see any visible slow-down on ARM neither in boot or in 
> > network
> > performance (I didn't run very sophisticated benchmarks).
>  
> What do you mean by "non-pvh aware xl"? All useful versions of xl
> support pvh?


I mean an xl without our PVH patches merged.
xl in upstream doesn't know much about PVH yet.
Even for ARM, we're still carrying significant patches in our tree.


> > > later on with the precise vCPU value which should be provided to QEMU
> > > via the -smp command line option
> > > (tools/libs/light/libxl_dm.c:libxl__build_device_model_args_new)?
> > 
> > Yes, a pvh aware xl will for example pass -smp 2,maxcpus=4 based on
> > values from the xl.cfg. If the user doesn't set maxvcpus in xl.cfg, xl
> > will set maxvcpus to the same value as vcpus.
> 
> OK good. In that case if this is just an initial value meant to be
> overwritten, I think it is best to keep it as 1.

Sorry but that won't work. I think the confusion here may be that
it's easy to mix up mc->max_cpus and machine->smp.max_cpus, these are
not the same. They have different purposes.

I'll try to clarify the 3 values in play.

machine-smp.cpus:
Number of guest vcpus active at boot.
Passed to QEMU via the -smp command-line option.
We don't use this value in QEMU's ARM PVH machines.

machine->smp.max_cpus:
Max number of vcpus that the guest can use (equal or larger than 
machine-smp.cpus).
Will be set by xl via the "-smp X,maxcpus=Y" command-line option to QEMU.
Taken from maxvcpus from xl.cfg, same as XEN_DMOP_nr_vcpus.
This is what we use for xen_register_ioreq().

mc->max_cpus:
Absolute MAX in QEMU used to cap the -smp command-line options.
If xl tries to set -smp (machine->smp.max_cpus) larger than this, QEMU will 
bail out.
Used to setup xen_register_ioreq() ONLY if -smp maxcpus was NOT set (i.e by a 
non PVH aware xl).
Cannot be 1 because that would limit QEMU to MAX 1 vcpu.

I guess we could set mc->max_cpus to what XEN_DMOP_nr_vcpus returns but I'll
have to check if we can even issue that hypercall this early in QEMU since
mc->max_cpus is setup before we even parse the machine options. We may
not yet know what domid we're attaching to yet.

Cheers,
Edgar

Re: [PATCH v1 04/10] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

On Mon, Aug 12, 2024 at 06:47:17PM -0700, Stefano Stabellini wrote:
> On Mon, 12 Aug 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> > 
> > Add SMP support for Xen PVH ARM guests. Create max_cpus ioreq
> > servers to handle hotplug.
> > 
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/arm/xen_arm.c | 5 +++--
> >  1 file changed, 3 insertions(+), 2 deletions(-)
> > 
> > diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
> > index 5f75cc3779..ef8315969c 100644
> > --- a/hw/arm/xen_arm.c
> > +++ b/hw/arm/xen_arm.c
> > @@ -173,7 +173,7 @@ static void xen_arm_init(MachineState *machine)
> >  
> >  xen_init_ram(machine);
> >  
> > -xen_register_ioreq(xam->state, machine->smp.cpus, 
> > &xen_memory_listener);
> > +xen_register_ioreq(xam->state, machine->smp.max_cpus, 
> > &xen_memory_listener);
> >  
> >  xen_create_virtio_mmio_devices(xam);
> >  
> > @@ -218,7 +218,8 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
> > void *data)
> >  MachineClass *mc = MACHINE_CLASS(oc);
> >  mc->desc = "Xen PVH ARM machine";
> >  mc->init = xen_arm_init;
> > -mc->max_cpus = 1;
> > +/* MAX number of vcpus supported by Xen.  */
> > +mc->max_cpus = GUEST_MAX_VCPUS;
> 
> Will this cause allocations of data structures with 128 elements?
> Looking at hw/xen/xen-hvm-common.c:xen_do_ioreq_register it seems
> possible? Or hw/xen/xen-hvm-common.c:xen_do_ioreq_register is called

Yes, in theory there's probably overhead with this but as you correctly
noted below, a PVH aware xl will set the max_cpus option to a lower value.

With a non-pvh aware xl, I was a little worried about the overhead
but I couldn't see any visible slow-down on ARM neither in boot or in network
performance (I didn't run very sophisticated benchmarks).


> later on with the precise vCPU value which should be provided to QEMU
> via the -smp command line option
> (tools/libs/light/libxl_dm.c:libxl__build_device_model_args_new)?

Yes, a pvh aware xl will for example pass -smp 2,maxcpus=4 based on
values from the xl.cfg. If the user doesn't set maxvcpus in xl.cfg, xl
will set maxvcpus to the same value as vcpus.

Best regards,
Edgar

[PATCH v1 10/10] docs/system/i386: xenpvh: Add a basic description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
---
 MAINTAINERS |  1 +
 docs/system/i386/xenpvh.rst | 49 +
 docs/system/target-i386.rst |  1 +
 3 files changed, 51 insertions(+)
 create mode 100644 docs/system/i386/xenpvh.rst

diff --git a/MAINTAINERS b/MAINTAINERS
index a24c2e14d9..da4c9d4d46 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -560,6 +560,7 @@ F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
 F: docs/system/arm/xenpvh.rst
+F: docs/system/i386/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
diff --git a/docs/system/i386/xenpvh.rst b/docs/system/i386/xenpvh.rst
new file mode 100644
index 00..354250f073
--- /dev/null
+++ b/docs/system/i386/xenpvh.rst
@@ -0,0 +1,49 @@
+Xen PVH machine (``xenpvh``)
+=
+
+Xen supports a spectrum of types of guests that vary in how they depend
+on HW virtualization features, emulation models and paravirtualization.
+PVH is a mode that uses HW virtualization features (like HVM) but tries
+to avoid emulation models and instead use passthrough or
+paravirtualized devices.
+
+QEMU can be used to provide PV virtio devices on an emulated PCIe controller.
+That is the purpose of this minimal machine.
+
+Supported devices
+-
+
+The x86 Xen PVH QEMU machine provide the following devices:
+
+- RAM
+- GPEX host bridge
+- virtio-pci devices
+
+The idea is to only connect virtio-pci devices but in theory any compatible
+PCI device model will work depending on Xen and guest support.
+
+Running
+---
+
+The Xen tools will typically construct a command-line and launch QEMU
+for you when needed. But here's an example of what it can look like in
+case you need to construct one manually:
+
+.. code-block:: console
+
+qemu-system-i386 -xen-domid 3 -no-shutdown\
+  -chardev 
socket,id=libxl-cmd,path=/var/run/xen/qmp-libxl-3,server=on,wait=off \
+  -mon chardev=libxl-cmd,mode=control \
+  -chardev 
socket,id=libxenstat-cmd,path=/var/run/xen/qmp-libxenstat-3,server=on,wait=off \
+  -mon chardev=libxenstat-cmd,mode=control\
+  -nodefaults \
+  -no-user-config \
+  -xen-attach -name g0\
+  -vnc none   \
+  -display none   \
+  -device virtio-net-pci,id=nic0,netdev=net0,mac=00:16:3e:5c:81:78 \
+  -netdev 
type=tap,id=net0,ifname=vif3.0-emu,br=xenbr0,script=no,downscript=no \
+  -smp 4,maxcpus=4\
+  -nographic  \
+  -machine 
xenpvh,ram-low-base=0,ram-low-size=2147483648,ram-high-base=4294967296,ram-high-size=2147483648,pci-ecam-base=824633720832,pci-ecam-size=268435456,pci-mmio-base=4026531840,pci-mmio-size=33554432,pci-mmio-high-base=824902156288,pci-mmio-high-size=68719476736
 \
+  -m 4096
diff --git a/docs/system/target-i386.rst b/docs/system/target-i386.rst
index 1b8a1f248a..23e84e3ba7 100644
--- a/docs/system/target-i386.rst
+++ b/docs/system/target-i386.rst
@@ -26,6 +26,7 @@ Architectural features
i386/cpu
i386/hyperv
i386/xen
+   i386/xenpvh
i386/kvm-pv
i386/sgx
i386/amd-memory-encryption
-- 
2.43.0

[PATCH v1 08/10] hw/xen: pvh-common: Add support for creating PCIe/GPEX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add support for optionally creating a PCIe/GPEX controller.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-pvh-common.c | 66 +
 include/hw/xen/xen-pvh-common.h | 10 -
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 69a2dbdb6d..b1432e4bd9 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -120,6 +120,59 @@ static void xen_enable_tpm(XenPVHCommonState *s)
 }
 #endif
 
+static void xen_set_pci_intx_irq(void *opaque, int irq, int level)
+{
+if (xen_set_pci_intx_level(xen_domid, 0, 0, 0, irq, level)) {
+error_report("xendevicemodel_set_pci_intx_level failed");
+}
+}
+
+static inline void xenpvh_gpex_init(XenPVHCommonState *s,
+MemoryRegion *sysmem,
+hwaddr ecam_base, hwaddr ecam_size,
+hwaddr mmio_base, hwaddr mmio_size,
+hwaddr mmio_high_base,
+hwaddr mmio_high_size,
+int intx_irq_base)
+{
+MemoryRegion *ecam_reg;
+MemoryRegion *mmio_reg;
+DeviceState *dev;
+int i;
+
+object_initialize_child(OBJECT(s), "gpex", &s->pci.gpex,
+TYPE_GPEX_HOST);
+dev = DEVICE(&s->pci.gpex);
+sysbus_realize_and_unref(SYS_BUS_DEVICE(dev), &error_fatal);
+
+ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
+memory_region_add_subregion(sysmem, ecam_base, ecam_reg);
+
+mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
+
+if (mmio_size) {
+memory_region_init_alias(&s->pci.mmio_alias, OBJECT(dev), "pcie-mmio",
+ mmio_reg, mmio_base, mmio_size);
+memory_region_add_subregion(sysmem, mmio_base, &s->pci.mmio_alias);
+}
+
+if (mmio_high_size) {
+memory_region_init_alias(&s->pci.mmio_high_alias, OBJECT(dev),
+"pcie-mmio-high",
+mmio_reg, mmio_high_base, mmio_high_size);
+memory_region_add_subregion(sysmem, mmio_high_base,
+&s->pci.mmio_high_alias);
+}
+
+for (i = 0; i < GPEX_NUM_IRQS; i++) {
+qemu_irq irq = qemu_allocate_irq(xen_set_pci_intx_irq, s, i);
+
+sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, irq);
+gpex_set_irq_num(GPEX_HOST(dev), i, intx_irq_base + i);
+xen_set_pci_link_route(i, intx_irq_base + i);
+}
+}
+
 static void xen_pvh_realize(DeviceState *dev, Error **errp)
 {
 XenPVHCommonState *s = XEN_PVH_COMMON(dev);
@@ -152,6 +205,14 @@ static void xen_pvh_realize(DeviceState *dev, Error **errp)
 warn_report("tpm-base-addr is not provided. TPM will not be enabled");
 }
 #endif
+
+if (s->cfg.ecam.size) {
+xenpvh_gpex_init(s, sysmem,
+ s->cfg.ecam.base, s->cfg.ecam.size,
+ s->cfg.mmio.base, s->cfg.mmio.size,
+ s->cfg.mmio_high.base, s->cfg.mmio_high.size,
+ s->cfg.pci_intx_irq_base);
+}
 }
 
 #define DEFINE_PROP_MEMMAP(n, f) \
@@ -165,10 +226,15 @@ static Property xen_pvh_properties[] = {
 DEFINE_PROP_MEMMAP("ram-high", ram_high),
 DEFINE_PROP_MEMMAP("virtio-mmio", virtio_mmio),
 DEFINE_PROP_MEMMAP("tpm", tpm),
+DEFINE_PROP_MEMMAP("pci-ecam", ecam),
+DEFINE_PROP_MEMMAP("pci-mmio", mmio),
+DEFINE_PROP_MEMMAP("pci-mmio-high", mmio_high),
 DEFINE_PROP_UINT32("virtio-mmio-num", XenPVHCommonState,
cfg.virtio_mmio_num, 0),
 DEFINE_PROP_UINT32("virtio-mmio-irq-base", XenPVHCommonState,
cfg.virtio_mmio_irq_base, 0),
+DEFINE_PROP_UINT32("pci-intx-irq-base", XenPVHCommonState,
+   cfg.pci_intx_irq_base, 0),
 DEFINE_PROP_END_OF_LIST()
 };
 
diff --git a/include/hw/xen/xen-pvh-common.h b/include/hw/xen/xen-pvh-common.h
index e958b441fd..faacfca70e 100644
--- a/include/hw/xen/xen-pvh-common.h
+++ b/include/hw/xen/xen-pvh-common.h
@@ -29,17 +29,25 @@ typedef struct XenPVHCommonState {
 MemoryRegion high;
 } ram;
 
+struct {
+GPEXHost gpex;
+MemoryRegion mmio_alias;
+MemoryRegion mmio_high_alias;
+} pci;
+
 struct {
 uint64_t ram_size;
 uint32_t max_cpus;
 uint32_t virtio_mmio_num;
 uint32_t virtio_mmio_irq_base;
+uint32_t pci_intx_irq_base;
 struct {
 uint64_t base;
 uint64_t size;
 } ram_low, ram_high,
   virtio_mmio,
-  tpm;
+  tpm,
+  ecam, mmio, mmio_high;
 } cfg;
 } XenPVHCommonState;
 #endif
-- 
2.43.0

[PATCH v1 04/10] hw/arm: xenpvh: Add support for SMP guests

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add SMP support for Xen PVH ARM guests. Create max_cpus ioreq
servers to handle hotplug.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/xen_arm.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 5f75cc3779..ef8315969c 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -173,7 +173,7 @@ static void xen_arm_init(MachineState *machine)
 
 xen_init_ram(machine);
 
-xen_register_ioreq(xam->state, machine->smp.cpus, &xen_memory_listener);
+xen_register_ioreq(xam->state, machine->smp.max_cpus, 
&xen_memory_listener);
 
 xen_create_virtio_mmio_devices(xam);
 
@@ -218,7 +218,8 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 MachineClass *mc = MACHINE_CLASS(oc);
 mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
-mc->max_cpus = 1;
+/* MAX number of vcpus supported by Xen.  */
+mc->max_cpus = GUEST_MAX_VCPUS;
 mc->default_machine_opts = "accel=xen";
 /* Set explicitly here to make sure that real ram_size is passed */
 mc->default_ram_size = 0;
-- 
2.43.0

[PATCH v1 02/10] hw/arm: xenpvh: Update file header to use SPDX

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Update file header to use SPDX and remove stray empty
comment line.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/xen_arm.c | 19 +--
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 6fad829ede..766a194fa1 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -1,24 +1,7 @@
 /*
  * QEMU ARM Xen PVH Machine
  *
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to 
deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
+ * SPDX-License-Identifier: MIT
  */
 
 #include "qemu/osdep.h"
-- 
2.43.0

[PATCH v1 06/10] hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Rename xen_arm.c -> xen-pvh.c to better express that this
is a PVH machine and to align with x86 HVM and future PVH
machine filenames:
hw/i386/xen/xen-hvm.c
hw/i386/xen/xen-pvh.c (in preparation)

No functional changes.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/meson.build  | 2 +-
 hw/arm/{xen_arm.c => xen-pvh.c} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename hw/arm/{xen_arm.c => xen-pvh.c} (100%)

diff --git a/hw/arm/meson.build b/hw/arm/meson.build
index 0c07ab522f..769fe9ec1a 100644
--- a/hw/arm/meson.build
+++ b/hw/arm/meson.build
@@ -59,7 +59,7 @@ arm_ss.add(when: 'CONFIG_FSL_IMX7', if_true: 
files('fsl-imx7.c', 'mcimx7d-sabre.
 arm_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmuv3.c'))
 arm_ss.add(when: 'CONFIG_FSL_IMX6UL', if_true: files('fsl-imx6ul.c', 
'mcimx6ul-evk.c'))
 arm_ss.add(when: 'CONFIG_NRF51_SOC', if_true: files('nrf51_soc.c'))
-arm_ss.add(when: 'CONFIG_XEN', if_true: files('xen_arm.c'))
+arm_ss.add(when: 'CONFIG_XEN', if_true: files('xen-pvh.c'))
 
 system_ss.add(when: 'CONFIG_ARM_SMMUV3', if_true: files('smmu-common.c'))
 system_ss.add(when: 'CONFIG_CHEETAH', if_true: files('palm.c'))
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen-pvh.c
similarity index 100%
rename from hw/arm/xen_arm.c
rename to hw/arm/xen-pvh.c
-- 
2.43.0

[PATCH v1 09/10] hw/i386/xen: Add a Xen PVH x86 machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This adds a Xen PVH x86 machine based on the PVH Common
module used by the ARM PVH machine.

Signed-off-by: Edgar E. Iglesias 
---
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 196 
 2 files changed, 197 insertions(+)
 create mode 100644 hw/i386/xen/xen-pvh.c

diff --git a/hw/i386/xen/meson.build b/hw/i386/xen/meson.build
index 3f0df8bc07..c73c62b8e3 100644
--- a/hw/i386/xen/meson.build
+++ b/hw/i386/xen/meson.build
@@ -4,6 +4,7 @@ i386_ss.add(when: 'CONFIG_XEN', if_true: files(
 ))
 i386_ss.add(when: ['CONFIG_XEN', xen], if_true: files(
   'xen-hvm.c',
+  'xen-pvh.c',
 ))
 
 i386_ss.add(when: 'CONFIG_XEN_BUS', if_true: files(
diff --git a/hw/i386/xen/xen-pvh.c b/hw/i386/xen/xen-pvh.c
new file mode 100644
index 00..9c3d3fc58d
--- /dev/null
+++ b/hw/i386/xen/xen-pvh.c
@@ -0,0 +1,196 @@
+/*
+ * QEMU Xen PVH x86 Machine
+ *
+ * Copyright (c) 2024 Advanced Micro Devices, Inc.
+ * Written by Edgar E. Iglesias 
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+#include "qemu/error-report.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "hw/xen/arch_hvm.h"
+#include "hw/xen/xen.h"
+#include "hw/xen/xen-pvh-common.h"
+
+#define TYPE_XEN_PVH_X86  MACHINE_TYPE_NAME("xenpvh")
+OBJECT_DECLARE_SIMPLE_TYPE(XenPVHx86State, XEN_PVH_X86)
+
+#define PVH_MAX_CPUS 128
+
+struct XenPVHx86State {
+/*< private >*/
+MachineState parent;
+
+DeviceState *cpu[PVH_MAX_CPUS];
+XenPVHCommonState pvh;
+
+/*
+ * We provide these properties to allow Xen to move things to other
+ * addresses for example when users need to accomodate the memory-map
+ * for 1:1 mapped devices/memory.
+ */
+struct {
+MemMapEntry ram_low, ram_high;
+MemMapEntry pci_ecam, pci_mmio, pci_mmio_high;
+} cfg;
+};
+
+static void xenpvh_cpu_new(MachineState *ms,
+   XenPVHx86State *xp,
+   int cpu_idx,
+   int64_t apic_id)
+{
+Object *cpu = object_new(ms->cpu_type);
+
+object_property_add_child(OBJECT(ms), "cpu[*]", cpu);
+object_property_set_uint(cpu, "apic-id", apic_id, &error_fatal);
+qdev_realize(DEVICE(cpu), NULL, &error_fatal);
+object_unref(cpu);
+
+xp->cpu[cpu_idx] = DEVICE(cpu);
+}
+
+static void xenpvh_init(MachineState *ms)
+{
+XenPVHx86State *xp = XEN_PVH_X86(ms);
+const struct {
+const char *name;
+MemMapEntry *map;
+} map[] = {
+{ "ram-low", &xp->cfg.ram_low },
+{ "ram-high", &xp->cfg.ram_high },
+{ "pci-ecam", &xp->cfg.pci_ecam },
+{ "pci-mmio", &xp->cfg.pci_mmio },
+{ "pci-mmio-high", &xp->cfg.pci_mmio_high },
+};
+int i;
+
+object_initialize_child(OBJECT(ms), "pvh", &xp->pvh, TYPE_XEN_PVH_COMMON);
+object_property_set_int(OBJECT(&xp->pvh), "max-cpus", ms->smp.max_cpus,
+&error_abort);
+object_property_set_int(OBJECT(&xp->pvh), "ram-size", ms->ram_size,
+&error_abort);
+
+for (i = 0; i < ARRAY_SIZE(map); i++) {
+g_autofree char *base_name = g_strdup_printf("%s-base", map[i].name);
+g_autofree char *size_name = g_strdup_printf("%s-size", map[i].name);
+
+object_property_set_int(OBJECT(&xp->pvh), base_name, map[i].map->base,
+ &error_abort);
+object_property_set_int(OBJECT(&xp->pvh), size_name, map[i].map->size,
+ &error_abort);
+}
+
+/* GSI's 16 - 20 are used for legacy PCIe INTX IRQs.  */
+object_property_set_int(OBJECT(&xp->pvh), "pci-intx-irq-base", 16,
+&error_abort);
+
+sysbus_realize(SYS_BUS_DEVICE(&xp->pvh), &error_abort);
+
+/* Create dummy cores. This will indirectly create the APIC MSI window.  */
+for (i = 0; i < ms->smp.cpus; i++) {
+xenpvh_cpu_new(ms, xp, i, i);
+}
+}
+
+#define XENPVH_PROP_MEMMAP_SETTER(n, f)\
+static void xenpvh_set_ ## n ## _ ## f(Object *obj, Visitor *v,\
+   const char *name, void *opaque, \
+   Error **errp)   \
+{  \
+XenPVHx86State *xp = XEN_PVH_X86(obj); 

[PATCH v1 07/10] hw/arm: xenpvh: Reverse virtio-mmio creation order

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

We've been creating the virtio-mmio devices in forwards order
but since the qbus lists prepend (rather than append) entries,
the virtio busses end up with decreasing base address order.

Xen enables virtio-mmio nodes in forwards order so there's been
a missmatch. So far, we've been working around this with an
out-of-tree patch to Xen.

This reverses the order making sure the virtio busses end up
ordered with increasing base addresses avoiding the need to
patch Xen.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-pvh-common.c | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-pvh-common.c b/hw/xen/xen-pvh-common.c
index 0d368398d0..69a2dbdb6d 100644
--- a/hw/xen/xen-pvh-common.c
+++ b/hw/xen/xen-pvh-common.c
@@ -73,7 +73,18 @@ static void xen_create_virtio_mmio_devices(XenPVHCommonState 
*s)
 {
 int i;
 
-for (i = 0; i < s->cfg.virtio_mmio_num; i++) {
+/*
+ * We create the transports in reverse order. Since qbus_realize()
+ * prepends (not appends) new child buses, the decrementing loop below will
+ * create a list of virtio-mmio buses with increasing base addresses.
+ *
+ * When a -device option is processed from the command line,
+ * qbus_find_recursive() picks the next free virtio-mmio bus in forwards
+ * order.
+ *
+ * This is what the Xen tools expect.
+ */
+for (i = s->cfg.virtio_mmio_num - 1; i >= 0; i--) {
 hwaddr base = s->cfg.virtio_mmio.base + i * s->cfg.virtio_mmio.size;
 qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
  s->cfg.virtio_mmio_irq_base + i);
-- 
2.43.0

[PATCH v1 00/10] xen: pvh: Partial QOM:fication with new x86 PVH machine

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This series breaks-out parts of the ARM PVH support into a reusable
QOM module. There's a bit of refactoring and some bug-fixes along
the way.

Finally we add a new x86 xen-pvh machine using the new xen-pvh-common
module.

The corresponding changes Xen for PVH x86 are work in progress
(by Xenia Ragiadakou). You can find a current version here:
https://github.com/edgarigl/xen/tree/edgar/virtio-pvh-upstream 

I've briefly described the steps to run Xen PVH x86 guests here
(including example guest kernel config, xl.cfg and xl command-lines):
https://github.com/edgarigl/docs/blob/master/xen/pvh/xenpvh-x86.md

Cheers,
Edgar

Edgar E. Iglesias (10):
  MAINTAINERS: Add docs/system/arm/xenpvh.rst
  hw/arm: xenpvh: Update file header to use SPDX
  hw/arm: xenpvh: Tweak machine description
  hw/arm: xenpvh: Add support for SMP guests
  hw/arm: xenpvh: Break out a common PVH module
  hw/arm: xenpvh: Rename xen_arm.c -> xen-pvh.c
  hw/arm: xenpvh: Reverse virtio-mmio creation order
  hw/xen: pvh-common: Add support for creating PCIe/GPEX
  hw/i386/xen: Add a Xen PVH x86 machine
  docs/system/i386: xenpvh: Add a basic description

 MAINTAINERS |   2 +
 docs/system/i386/xenpvh.rst |  49 ++
 docs/system/target-i386.rst |   1 +
 hw/arm/meson.build  |   2 +-
 hw/arm/trace-events |   5 -
 hw/arm/xen-pvh.c| 165 
 hw/arm/xen_arm.c| 267 
 hw/i386/xen/meson.build |   1 +
 hw/i386/xen/xen-pvh.c   | 196 +++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 262 +++
 include/hw/xen/xen-pvh-common.h |  53 +++
 13 files changed, 735 insertions(+), 273 deletions(-)
 create mode 100644 docs/system/i386/xenpvh.rst
 create mode 100644 hw/arm/xen-pvh.c
 delete mode 100644 hw/arm/xen_arm.c
 create mode 100644 hw/i386/xen/xen-pvh.c
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

-- 
2.43.0

[PATCH v1 05/10] hw/arm: xenpvh: Break out a common PVH module

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Break out a common Xen PVH module in preparation for
adding a x86 Xen PVH Machine.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/trace-events |   5 -
 hw/arm/xen_arm.c| 154 ++
 hw/xen/meson.build  |   1 +
 hw/xen/trace-events |   4 +
 hw/xen/xen-pvh-common.c | 185 
 include/hw/xen/xen-pvh-common.h |  45 
 6 files changed, 269 insertions(+), 125 deletions(-)
 create mode 100644 hw/xen/xen-pvh-common.c
 create mode 100644 include/hw/xen/xen-pvh-common.h

diff --git a/hw/arm/trace-events b/hw/arm/trace-events
index be6c8f720b..c64ad344bd 100644
--- a/hw/arm/trace-events
+++ b/hw/arm/trace-events
@@ -68,10 +68,5 @@ z2_aer915_send_too_long(int8_t msg) "message too long (%i 
bytes)"
 z2_aer915_send(uint8_t reg, uint8_t value) "reg %d value 0x%02x"
 z2_aer915_event(int8_t event, int8_t len) "i2c event =0x%x len=%d bytes"
 
-# xen_arm.c
-xen_create_virtio_mmio_devices(int i, int irq, uint64_t base) "Created 
virtio-mmio device %d: irq %d base 0x%"PRIx64
-xen_init_ram(uint64_t machine_ram_size) "Initialized xen ram with size 
0x%"PRIx64
-xen_enable_tpm(uint64_t addr) "Connected tpmdev at address 0x%"PRIx64
-
 # bcm2838.c
 bcm2838_gic_set_irq(int irq, int level) "gic irq:%d lvl:%d"
diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index ef8315969c..b8a5c09bdf 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -12,40 +12,25 @@
 #include "hw/irq.h"
 #include "hw/sysbus.h"
 #include "sysemu/block-backend.h"
-#include "sysemu/tpm_backend.h"
 #include "sysemu/sysemu.h"
-#include "hw/xen/xen-hvm-common.h"
+#include "hw/xen/xen-pvh-common.h"
 #include "sysemu/tpm.h"
 #include "hw/xen/arch_hvm.h"
-#include "trace.h"
 
 #define TYPE_XEN_ARM  MACHINE_TYPE_NAME("xenpvh")
 OBJECT_DECLARE_SIMPLE_TYPE(XenArmState, XEN_ARM)
 
-static const MemoryListener xen_memory_listener = {
-.region_add = xen_region_add,
-.region_del = xen_region_del,
-.log_start = NULL,
-.log_stop = NULL,
-.log_sync = NULL,
-.log_global_start = NULL,
-.log_global_stop = NULL,
-.priority = MEMORY_LISTENER_PRIORITY_ACCEL,
-};
-
 struct XenArmState {
 /*< private >*/
 MachineState parent;
 
-XenIOState *state;
+XenPVHCommonState pvh;
 
 struct {
 uint64_t tpm_base_addr;
 } cfg;
 };
 
-static MemoryRegion ram_lo, ram_hi;
-
 /*
  * VIRTIO_MMIO_DEV_SIZE is imported from tools/libs/light/libxl_arm.c under Xen
  * repository.
@@ -57,64 +42,6 @@ static MemoryRegion ram_lo, ram_hi;
 #define NR_VIRTIO_MMIO_DEVICES   \
(GUEST_VIRTIO_MMIO_SPI_LAST - GUEST_VIRTIO_MMIO_SPI_FIRST)
 
-static void xen_set_irq(void *opaque, int irq, int level)
-{
-if (xendevicemodel_set_irq_level(xen_dmod, xen_domid, irq, level)) {
-error_report("xendevicemodel_set_irq_level failed");
-}
-}
-
-static void xen_create_virtio_mmio_devices(XenArmState *xam)
-{
-int i;
-
-for (i = 0; i < NR_VIRTIO_MMIO_DEVICES; i++) {
-hwaddr base = GUEST_VIRTIO_MMIO_BASE + i * VIRTIO_MMIO_DEV_SIZE;
-qemu_irq irq = qemu_allocate_irq(xen_set_irq, NULL,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i);
-
-sysbus_create_simple("virtio-mmio", base, irq);
-
-trace_xen_create_virtio_mmio_devices(i,
- GUEST_VIRTIO_MMIO_SPI_FIRST + i,
- base);
-}
-}
-
-static void xen_init_ram(MachineState *machine)
-{
-MemoryRegion *sysmem = get_system_memory();
-ram_addr_t block_len, ram_size[GUEST_RAM_BANKS];
-
-trace_xen_init_ram(machine->ram_size);
-if (machine->ram_size <= GUEST_RAM0_SIZE) {
-ram_size[0] = machine->ram_size;
-ram_size[1] = 0;
-block_len = GUEST_RAM0_BASE + ram_size[0];
-} else {
-ram_size[0] = GUEST_RAM0_SIZE;
-ram_size[1] = machine->ram_size - GUEST_RAM0_SIZE;
-block_len = GUEST_RAM1_BASE + ram_size[1];
-}
-
-memory_region_init_ram(&xen_memory, NULL, "xen.ram", block_len,
-   &error_fatal);
-
-memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo", &xen_memory,
- GUEST_RAM0_BASE, ram_size[0]);
-memory_region_add_subregion(sysmem, GUEST_RAM0_BASE, &ram_lo);
-if (ram_size[1] > 0) {
-memory_region_init_alias(&ram_hi, NULL, "xen.ram.hi", &xen_memory,
- GUEST_RAM1_BASE, ram_size[1]);
-memory_region_add_subregion(sysmem, GUEST_RAM1_BASE, &ram_hi);
-}
-
-/* Setup support for grants.  */
-memory_region_init_ram(&xen

[PATCH v1 03/10] hw/arm: xenpvh: Tweak machine description

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Tweak machine description to better express that this is
a Xen PVH machine for ARM.

Signed-off-by: Edgar E. Iglesias 
---
 hw/arm/xen_arm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hw/arm/xen_arm.c b/hw/arm/xen_arm.c
index 766a194fa1..5f75cc3779 100644
--- a/hw/arm/xen_arm.c
+++ b/hw/arm/xen_arm.c
@@ -216,7 +216,7 @@ static void xen_arm_machine_class_init(ObjectClass *oc, 
void *data)
 {
 
 MachineClass *mc = MACHINE_CLASS(oc);
-mc->desc = "Xen Para-virtualized PC";
+mc->desc = "Xen PVH ARM machine";
 mc->init = xen_arm_init;
 mc->max_cpus = 1;
 mc->default_machine_opts = "accel=xen";
-- 
2.43.0

[PATCH v1 01/10] MAINTAINERS: Add docs/system/arm/xenpvh.rst

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Signed-off-by: Edgar E. Iglesias 
---
 MAINTAINERS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 10af212632..a24c2e14d9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -559,6 +559,7 @@ F: include/hw/xen/
 F: include/sysemu/xen.h
 F: include/sysemu/xen-mapcache.h
 F: stubs/xen-hw-stub.c
+F: docs/system/arm/xenpvh.rst
 
 Guest CPU Cores (NVMM)
 --
-- 
2.43.0

Re: [PATCH v1 1/1] xen/arm: smmuv3: Mark more init-only functions with __init

[Edgar E. Iglesias]

On Wed, Jun 5, 2024 at 11:55 AM Rahul Singh  wrote:

> Hi Edgar,
>
> > On 22 May 2024, at 2:28 PM, Edgar E. Iglesias 
> wrote:
> >
> > From: "Edgar E. Iglesias" 
> >
> > Move more functions that are only called at init to
> > the .init.text section.
> >
> > Signed-off-by: Edgar E. Iglesias 
>
> Acked-by: Rahul Singh 
> Tested-by: Rahul Singh 
>
> Regards,
> Rahul
>

Ping!

This still rebases cleanly on top of master but let me know if I should
resend.

Best regards,
Edgar

[PULL v1 3/3] xen: mapcache: Fix unmapping of first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This fixes the clobbering of the entry->next pointer when
unmapping the first entry in a bucket of a mapcache.

Fixes: 123acd816d ("xen: mapcache: Unmap first entries in buckets")
Reported-by: Anthony PERARD 
Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Anthony PERARD 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index 5f23b0adbe..18ba7b1d8f 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -597,7 +597,17 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 pentry->next = entry->next;
 g_free(entry);
 } else {
-memset(entry, 0, sizeof *entry);
+/*
+ * Invalidate mapping but keep entry->next pointing to the rest
+ * of the list.
+ *
+ * Note that lock is already zero here, otherwise we don't unmap.
+ */
+entry->paddr_index = 0;
+entry->vaddr_base = NULL;
+entry->valid_mapping = NULL;
+entry->flags = 0;
+entry->size = 0;
 }
 }
 
-- 
2.43.0

Re: [PATCH v1 2/2] xen: mapcache: Fix unmapping of first entries in buckets

[Edgar E. Iglesias]

On Thu, Jul 4, 2024 at 9:48 PM Edgar E. Iglesias 
wrote:

> On Thu, Jul 04, 2024 at 05:44:52PM +0100, Alex Bennée wrote:
> > Anthony PERARD  writes:
> >
> > > On Tue, Jul 02, 2024 at 12:44:21AM +0200, Edgar E. Iglesias wrote:
> > >> From: "Edgar E. Iglesias" 
> > >>
> > >> This fixes the clobbering of the entry->next pointer when
> > >> unmapping the first entry in a bucket of a mapcache.
> > >>
> > >> Fixes: 123acd816d ("xen: mapcache: Unmap first entries in buckets")
> > >> Reported-by: Anthony PERARD 
> > >> Signed-off-by: Edgar E. Iglesias 
> > >> ---
> > >>  hw/xen/xen-mapcache.c | 12 +++-
> > >>  1 file changed, 11 insertions(+), 1 deletion(-)
> > >>
> > >> diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > >> index 5f23b0adbe..18ba7b1d8f 100644
> > >> --- a/hw/xen/xen-mapcache.c
> > >> +++ b/hw/xen/xen-mapcache.c
> > >> @@ -597,7 +597,17 @@ static void
> xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
> > >>  pentry->next = entry->next;
> > >>  g_free(entry);
> > >>  } else {
> > >> -memset(entry, 0, sizeof *entry);
> > >> +/*
> > >> + * Invalidate mapping but keep entry->next pointing to the
> rest
> > >> + * of the list.
> > >> + *
> > >> + * Note that lock is already zero here, otherwise we don't
> unmap.
> > >> + */
> > >> +entry->paddr_index = 0;
> > >> +entry->vaddr_base = NULL;
> > >> +entry->valid_mapping = NULL;
> > >> +entry->flags = 0;
> > >> +entry->size = 0;
> > >
> > > This kind of feels like mc->entry should be an array of pointer rather
> > > than an array of MapCacheEntry but that seems to work well enough and
> > > not the first time entries are been cleared like that.
> >
> > The use of a hand rolled list is a bit of a concern considering QEMU and
> > Glib both provide various abstractions used around the rest of the code
> > base. The original patch that introduces the mapcache doesn't tell me
> > much about access patterns for the cache, just that it is trying to
> > solve memory exhaustion issues with lots of dynamic small mappings.
> >
> > Maybe a simpler structure is desirable?
> >
> > We also have an interval tree implementation ("qemu/interval-tree.h") if
> > what we really want is a sorted tree of memory that can be iterated
> > locklessly.
> >
>
> Yes, it would be interesting to benchmark other options.
> I agree that we should at minimum reuse existing lists/hash tables.
>
> We've also had some discussions around removing it partially or
> alltogether but
> there are some concerns around that. We're going to need something to
> keep track of grants. For 32-bit hosts, it's a problem to exhaust virtual
> address-space if mapping all of the guest (are folks still using 32-bit
> hosts?).
> There may be other issues aswell.
>
> Some benefits are that we'll remove some of the complexity and latency for
> mapping
> and unmapping stuff continously.
>
>
One more thing I forgot to add is that IMO, these larger longer term
changes should not block this tiny bugfix...

Cheers,
Edgar

Re: [PATCH v1 2/2] xen: mapcache: Fix unmapping of first entries in buckets

[Edgar E. Iglesias]

On Thu, Jul 04, 2024 at 05:44:52PM +0100, Alex Bennée wrote:
> Anthony PERARD  writes:
> 
> > On Tue, Jul 02, 2024 at 12:44:21AM +0200, Edgar E. Iglesias wrote:
> >> From: "Edgar E. Iglesias" 
> >> 
> >> This fixes the clobbering of the entry->next pointer when
> >> unmapping the first entry in a bucket of a mapcache.
> >> 
> >> Fixes: 123acd816d ("xen: mapcache: Unmap first entries in buckets")
> >> Reported-by: Anthony PERARD 
> >> Signed-off-by: Edgar E. Iglesias 
> >> ---
> >>  hw/xen/xen-mapcache.c | 12 +++-
> >>  1 file changed, 11 insertions(+), 1 deletion(-)
> >> 
> >> diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> >> index 5f23b0adbe..18ba7b1d8f 100644
> >> --- a/hw/xen/xen-mapcache.c
> >> +++ b/hw/xen/xen-mapcache.c
> >> @@ -597,7 +597,17 @@ static void 
> >> xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
> >>  pentry->next = entry->next;
> >>  g_free(entry);
> >>  } else {
> >> -memset(entry, 0, sizeof *entry);
> >> +/*
> >> + * Invalidate mapping but keep entry->next pointing to the rest
> >> + * of the list.
> >> + *
> >> + * Note that lock is already zero here, otherwise we don't unmap.
> >> + */
> >> +entry->paddr_index = 0;
> >> +entry->vaddr_base = NULL;
> >> +entry->valid_mapping = NULL;
> >> +entry->flags = 0;
> >> +entry->size = 0;
> >
> > This kind of feels like mc->entry should be an array of pointer rather
> > than an array of MapCacheEntry but that seems to work well enough and
> > not the first time entries are been cleared like that.
> 
> The use of a hand rolled list is a bit of a concern considering QEMU and
> Glib both provide various abstractions used around the rest of the code
> base. The original patch that introduces the mapcache doesn't tell me
> much about access patterns for the cache, just that it is trying to
> solve memory exhaustion issues with lots of dynamic small mappings.
> 
> Maybe a simpler structure is desirable?
> 
> We also have an interval tree implementation ("qemu/interval-tree.h") if
> what we really want is a sorted tree of memory that can be iterated
> locklessly.
> 

Yes, it would be interesting to benchmark other options.
I agree that we should at minimum reuse existing lists/hash tables.

We've also had some discussions around removing it partially or alltogether but
there are some concerns around that. We're going to need something to
keep track of grants. For 32-bit hosts, it's a problem to exhaust virtual
address-space if mapping all of the guest (are folks still using 32-bit hosts?).
There may be other issues aswell.

Some benefits are that we'll remove some of the complexity and latency for 
mapping
and unmapping stuff continously.

Cheers,
Edgar

Re: [PATCH v1 1/2] physmem: Bail out qemu_ram_block_from_host() for invalid ram addrs

[Edgar E. Iglesias]

On Thu, Jul 4, 2024 at 1:26 PM Alex Bennée  wrote:

> "Edgar E. Iglesias"  writes:
>
> > From: "Edgar E. Iglesias" 
> >
> > Bail out in qemu_ram_block_from_host() when
> > xen_ram_addr_from_mapcache() does not find an existing
> > mapping.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  system/physmem.c | 4 
> >  1 file changed, 4 insertions(+)
> >
> > diff --git a/system/physmem.c b/system/physmem.c
> > index 33d09f7571..59d1576c2b 100644
> > --- a/system/physmem.c
> > +++ b/system/physmem.c
> > @@ -2277,6 +2277,10 @@ RAMBlock *qemu_ram_block_from_host(void *ptr,
> bool round_offset,
> >  ram_addr_t ram_addr;
> >  RCU_READ_LOCK_GUARD();
> >  ram_addr = xen_ram_addr_from_mapcache(ptr);
> > +if (ram_addr == RAM_ADDR_INVALID) {
> > +return NULL;
> > +}
> > +
>
> Isn't this indicative of a failure? Should there at least be a trace
> point for failed mappings?
>
>
Yes but there are already trace points for the failure cases inside
xen_ram_addr_from_mapcache().
Do those address your concerns or do you think we need additional trace
points?

Cheers,
Edgar


> >  block = qemu_get_ram_block(ram_addr);
> >  if (block) {
> >  *offset = ram_addr - block->offset;
>
> --
> Alex Bennée
> Virtualisation Tech Lead @ Linaro
>

[PATCH v1 1/2] physmem: Bail out qemu_ram_block_from_host() for invalid ram addrs

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Bail out in qemu_ram_block_from_host() when
xen_ram_addr_from_mapcache() does not find an existing
mapping.

Signed-off-by: Edgar E. Iglesias 
---
 system/physmem.c | 4 
 1 file changed, 4 insertions(+)

diff --git a/system/physmem.c b/system/physmem.c
index 33d09f7571..59d1576c2b 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2277,6 +2277,10 @@ RAMBlock *qemu_ram_block_from_host(void *ptr, bool 
round_offset,
 ram_addr_t ram_addr;
 RCU_READ_LOCK_GUARD();
 ram_addr = xen_ram_addr_from_mapcache(ptr);
+if (ram_addr == RAM_ADDR_INVALID) {
+return NULL;
+}
+
 block = qemu_get_ram_block(ram_addr);
 if (block) {
 *offset = ram_addr - block->offset;
-- 
2.43.0

[PATCH v1 0/2] xen: mapcache: Fix unmapping of first the entry in a bucket

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This fixes the unmapping of the first mapping in a bucket of a mapcache.

We also add error handling to qemu_ram_block_from_host() to bail out when
xen_ram_addr_from_mapcache() doesn't find an existing mapping.

Cheers,
Edgar

Edgar E. Iglesias (2):
  physmem: Bail out qemu_ram_block_from_host() for invalid ram addrs
  xen: mapcache: Fix unmapping of first entries in buckets

 hw/xen/xen-mapcache.c | 12 +++-
 system/physmem.c  |  4 
 2 files changed, 15 insertions(+), 1 deletion(-)

-- 
2.43.0

[PATCH v1 2/2] xen: mapcache: Fix unmapping of first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

This fixes the clobbering of the entry->next pointer when
unmapping the first entry in a bucket of a mapcache.

Fixes: 123acd816d ("xen: mapcache: Unmap first entries in buckets")
Reported-by: Anthony PERARD 
Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-mapcache.c | 12 +++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index 5f23b0adbe..18ba7b1d8f 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -597,7 +597,17 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 pentry->next = entry->next;
 g_free(entry);
 } else {
-memset(entry, 0, sizeof *entry);
+/*
+ * Invalidate mapping but keep entry->next pointing to the rest
+ * of the list.
+ *
+ * Note that lock is already zero here, otherwise we don't unmap.
+ */
+entry->paddr_index = 0;
+entry->vaddr_base = NULL;
+entry->valid_mapping = NULL;
+entry->flags = 0;
+entry->size = 0;
 }
 }
 
-- 
2.43.0

Re: [PATCH v8 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

On Mon, Jul 1, 2024 at 6:21 PM Anthony PERARD 
wrote:

> On Mon, Jul 01, 2024 at 04:34:53PM +0200, Edgar E. Iglesias wrote:
> > On Mon, Jul 1, 2024 at 4:30 PM Edgar E. Iglesias <
> edgar.igles...@gmail.com>
> > wrote:
> > > On Mon, Jul 1, 2024 at 3:58 PM Edgar E. Iglesias <
> edgar.igles...@gmail.com>
> > > wrote:
> > >> Any chance you could try to get a backtrace from QEMU when it failed?
>
> Here it is:
>
>
> #3  0x7fa8762f4472 in __GI_abort () at ./stdlib/abort.c:79
> save_stage = 1
> act = {__sigaction_handler = {sa_handler = 0x20, sa_sigaction =
> 0x20}, sa_mask = {__val = {94603440166168, 18446744073709551615,
> 94603406369640, 0, 0, 94603406346720, 94603440166168, 140361486774256, 0,
> 140361486773376, 94603401285536, 140361496232688, 94603440166096,
> 140361486773456, 94603401289576, 140360849280256}}, sa_flags = -1804462896,
> sa_restorer = 0x748f2d40}
> #4  0x560a92230f0d in qemu_get_ram_block (addr=18446744073709551615)
> at ../system/physmem.c:801
> block = 0x0
> #5  0x560a922350ab in qemu_ram_block_from_host (ptr=0x7fa84e8fcd00,
> round_offset=false, offset=0x7fa8748f2de8) at ../system/physmem.c:2280
> ram_addr = 18446744073709551615
> _rcu_read_auto = 0x1
> block = 0x0
> host = 0x7fa84e8fcd00 ""
> _rcu_read_auto = 0x7fa8751f8288
> #6  0x560a92229669 in memory_region_from_host (ptr=0x7fa84e8fcd00,
> offset=0x7fa8748f2de8) at ../system/memory.c:2440
> block = 0x0
> #7  0x560a92237418 in address_space_unmap (as=0x560a94b05a20,
> buffer=0x7fa84e8fcd00, len=32768, is_write=true, access_len=32768) at
> ../system/physmem.c:3246
> mr = 0x0
> addr1 = 0
> __PRETTY_FUNCTION__ = "address_space_unmap"
> #8  0x560a91fd6cd3 in dma_memory_unmap (as=0x560a94b05a20,
> buffer=0x7fa84e8fcd00, len=32768, dir=DMA_DIRECTION_FROM_DEVICE,
> access_len=32768) at /root/build/qemu/include/sysemu/dma.h:236
> #9  0x560a91fd763d in dma_blk_unmap (dbs=0x560a94d87400) at
> ../system/dma-helpers.c:93
> i = 1
> #10 0x560a91fd76e6 in dma_complete (dbs=0x560a94d87400, ret=0) at
> ../system/dma-helpers.c:105
> __PRETTY_FUNCTION__ = "dma_complete"
> #11 0x560a91fd781c in dma_blk_cb (opaque=0x560a94d87400, ret=0) at
> ../system/dma-helpers.c:129
> dbs = 0x560a94d87400
> ctx = 0x560a9448da90
> cur_addr = 0
> cur_len = 0
> mem = 0x0
> __PRETTY_FUNCTION__ = "dma_blk_cb"
> #12 0x560a9232e974 in blk_aio_complete (acb=0x560a9448d5f0) at
> ../block/block-backend.c:1555
> #13 0x560a9232ebd1 in blk_aio_read_entry (opaque=0x560a9448d5f0) at
> ../block/block-backend.c:1610
> acb = 0x560a9448d5f0
> rwco = 0x560a9448d618
> qiov = 0x560a94d87460
> __PRETTY_FUNCTION__ = "blk_aio_read_entry"
>
> > > One more thing, regarding this specific patch. I don't think we should
> > > clear the
> > > entire entry, the next field should be kept, otherwise we'll disconnect
> > > following
> > > mappings that will never be found again. IIUC, this could very well be
> > > causing the problem you see.
> > >
> > > Does the following make sense?
> > >
> > And here without double-freeing entry->valid_mapping:
> >
> > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > index 5f23b0adbe..667807b3b6 100644
> > --- a/hw/xen/xen-mapcache.c
> > +++ b/hw/xen/xen-mapcache.c
> > @@ -597,7 +597,13 @@ static void
> > xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
> >  pentry->next = entry->next;
> >  g_free(entry);
> >  } else {
> > -memset(entry, 0, sizeof *entry);
> > +/* Invalidate mapping.  */
> > +entry->paddr_index = 0;
> > +entry->vaddr_base = NULL;
> > +entry->size = 0;
> > +entry->valid_mapping = NULL;
> > +entry->flags = 0;
> > +/* Keep entry->next pointing to the rest of the list.  */
> >  }
> >  }
>
> I've tried this patch, and that fix the issue I've seen. I'll run more
> tests on it, just in case, but there's no reason that would break
> something else.
>
> Cheers,
>

Thanks Anthony, I'll send out a proper patch tomorrow.

Cheers,
Edgar

Re: [PATCH v8 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

On Mon, Jul 1, 2024 at 4:30 PM Edgar E. Iglesias 
wrote:

>
>
> On Mon, Jul 1, 2024 at 3:58 PM Edgar E. Iglesias 
> wrote:
>
>> On Mon, Jul 1, 2024 at 2:55 PM Anthony PERARD 
>> wrote:
>>
>>> Hi all,
>>>
>>> Following this commit, a test which install Debian in a guest with OVMF
>>> as firmware started to fail. QEMU exit with an error when GRUB is
>>> running on the freshly installed Debian (I don't know if GRUB is
>>> starting Linux or not).
>>> The error is:
>>> Bad ram offset 
>>>
>>> Some logs:
>>>
>>> http://logs.test-lab.xenproject.org/osstest/logs/186611/test-amd64-amd64-xl-qemuu-ovmf-amd64/info.html
>>>
>>> Any idea? Something is trying to do something with the address "-1" when
>>> it shouldn't?
>>>
>>>
>> Hi Anothny,
>>
>> Yes, it looks like something is calling qemu_get_ram_block() on something
>> that isn't mapped.
>> One possible path is in qemu_ram_block_from_host() but there may be
>> others.
>>
>> The following patch may help.
>> Any chance you could try to get a backtrace from QEMU when it failed?
>>
>> diff --git a/system/physmem.c b/system/physmem.c
>> index 33d09f7571..2669c4dbbb 100644
>> --- a/system/physmem.c
>> +++ b/system/physmem.c
>> @@ -2277,6 +2277,9 @@ RAMBlock *qemu_ram_block_from_host(void *ptr, bool
>> round_offset,
>>  ram_addr_t ram_addr;
>>  RCU_READ_LOCK_GUARD();
>>  ram_addr = xen_ram_addr_from_mapcache(ptr);
>> +if (ram_addr == RAM_ADDR_INVALID) {
>> +return NULL;
>> +}
>>  block = qemu_get_ram_block(ram_addr);
>>  if (block) {
>>  *offset = ram_addr - block->offset;
>>
>>
>>
> One more thing, regarding this specific patch. I don't think we should
> clear the
> entire entry, the next field should be kept, otherwise we'll disconnect
> following
> mappings that will never be found again. IIUC, this could very well be
> causing the problem you see.
>
> Does the following make sense?
>
> diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> index 5f23b0adbe..e9df53c19d 100644
> --- a/hw/xen/xen-mapcache.c
> +++ b/hw/xen/xen-mapcache.c
> @@ -597,7 +597,14 @@ static void
> xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
>  pentry->next = entry->next;
>  g_free(entry);
>  } else {
> -memset(entry, 0, sizeof *entry);
> +/* Invalidate mapping.  */
> +entry->paddr_index = 0;
> +entry->vaddr_base = NULL;
> +entry->size = 0;
> +g_free(entry->valid_mapping);
> +entry->valid_mapping = NULL;
> +entry->flags = 0;
> +/* Keep entry->next pointing to the rest of the list.  */
>  }
>  }
>
>
>

And here without double-freeing entry->valid_mapping:

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index 5f23b0adbe..667807b3b6 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -597,7 +597,13 @@ static void
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 pentry->next = entry->next;
 g_free(entry);
 } else {
-memset(entry, 0, sizeof *entry);
+/* Invalidate mapping.  */
+entry->paddr_index = 0;
+entry->vaddr_base = NULL;
+entry->size = 0;
+entry->valid_mapping = NULL;
+entry->flags = 0;
+/* Keep entry->next pointing to the rest of the list.  */
 }
 }

Re: [PATCH v8 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

On Mon, Jul 1, 2024 at 3:58 PM Edgar E. Iglesias 
wrote:

> On Mon, Jul 1, 2024 at 2:55 PM Anthony PERARD 
> wrote:
>
>> Hi all,
>>
>> Following this commit, a test which install Debian in a guest with OVMF
>> as firmware started to fail. QEMU exit with an error when GRUB is
>> running on the freshly installed Debian (I don't know if GRUB is
>> starting Linux or not).
>> The error is:
>> Bad ram offset 
>>
>> Some logs:
>>
>> http://logs.test-lab.xenproject.org/osstest/logs/186611/test-amd64-amd64-xl-qemuu-ovmf-amd64/info.html
>>
>> Any idea? Something is trying to do something with the address "-1" when
>> it shouldn't?
>>
>>
> Hi Anothny,
>
> Yes, it looks like something is calling qemu_get_ram_block() on something
> that isn't mapped.
> One possible path is in qemu_ram_block_from_host() but there may be others.
>
> The following patch may help.
> Any chance you could try to get a backtrace from QEMU when it failed?
>
> diff --git a/system/physmem.c b/system/physmem.c
> index 33d09f7571..2669c4dbbb 100644
> --- a/system/physmem.c
> +++ b/system/physmem.c
> @@ -2277,6 +2277,9 @@ RAMBlock *qemu_ram_block_from_host(void *ptr, bool
> round_offset,
>  ram_addr_t ram_addr;
>  RCU_READ_LOCK_GUARD();
>  ram_addr = xen_ram_addr_from_mapcache(ptr);
> +if (ram_addr == RAM_ADDR_INVALID) {
> +return NULL;
> +}
>  block = qemu_get_ram_block(ram_addr);
>  if (block) {
>  *offset = ram_addr - block->offset;
>
>
>
One more thing, regarding this specific patch. I don't think we should
clear the
entire entry, the next field should be kept, otherwise we'll disconnect
following
mappings that will never be found again. IIUC, this could very well be
causing the problem you see.

Does the following make sense?

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index 5f23b0adbe..e9df53c19d 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -597,7 +597,14 @@ static void
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 pentry->next = entry->next;
 g_free(entry);
 } else {
-memset(entry, 0, sizeof *entry);
+/* Invalidate mapping.  */
+entry->paddr_index = 0;
+entry->vaddr_base = NULL;
+entry->size = 0;
+g_free(entry->valid_mapping);
+entry->valid_mapping = NULL;
+entry->flags = 0;
+/* Keep entry->next pointing to the rest of the list.  */
 }
 }









>
>
>> Cheers,
>>
>> Anthony
>>
>> On Wed, May 29, 2024 at 04:07:33PM +0200, Edgar E. Iglesias wrote:
>> > From: "Edgar E. Iglesias" 
>> >
>> > When invalidating memory ranges, if we happen to hit the first
>> > entry in a bucket we were never unmapping it. This was harmless
>> > for foreign mappings but now that we're looking to reuse the
>> > mapcache for transient grant mappings, we must unmap entries
>> > when invalidated.
>> >
>> > Signed-off-by: Edgar E. Iglesias 
>> > Reviewed-by: Stefano Stabellini 
>> > ---
>> >  hw/xen/xen-mapcache.c | 11 ---
>> >  1 file changed, 8 insertions(+), 3 deletions(-)
>> >
>> > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
>> > index bc860f4373..ec95445696 100644
>> > --- a/hw/xen/xen-mapcache.c
>> > +++ b/hw/xen/xen-mapcache.c
>> > @@ -491,18 +491,23 @@ static void
>> xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
>> >  return;
>> >  }
>> >  entry->lock--;
>> > -if (entry->lock > 0 || pentry == NULL) {
>> > +if (entry->lock > 0) {
>> >  return;
>> >  }
>> >
>> > -pentry->next = entry->next;
>> >  ram_block_notify_remove(entry->vaddr_base, entry->size,
>> entry->size);
>> >  if (munmap(entry->vaddr_base, entry->size) != 0) {
>> >  perror("unmap fails");
>> >  exit(-1);
>> >  }
>> > +
>> >  g_free(entry->valid_mapping);
>> > -g_free(entry);
>> > +if (pentry) {
>> > +pentry->next = entry->next;
>> > +g_free(entry);
>> > +} else {
>> > +memset(entry, 0, sizeof *entry);
>> > +}
>> >  }
>> >
>> >  typedef struct XenMapCacheData {
>> > --
>> > 2.40.1
>> >
>> >
>> --
>>
>> Anthony Perard | Vates XCP-ng Developer
>>
>> XCP-ng & Xen Orchestra - Vates solutions
>>
>> web: https://vates.tech
>>
>

Re: [PATCH v8 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

On Mon, Jul 1, 2024 at 2:55 PM Anthony PERARD 
wrote:

> Hi all,
>
> Following this commit, a test which install Debian in a guest with OVMF
> as firmware started to fail. QEMU exit with an error when GRUB is
> running on the freshly installed Debian (I don't know if GRUB is
> starting Linux or not).
> The error is:
> Bad ram offset 
>
> Some logs:
>
> http://logs.test-lab.xenproject.org/osstest/logs/186611/test-amd64-amd64-xl-qemuu-ovmf-amd64/info.html
>
> Any idea? Something is trying to do something with the address "-1" when
> it shouldn't?
>
>
Hi Anothny,

Yes, it looks like something is calling qemu_get_ram_block() on something
that isn't mapped.
One possible path is in qemu_ram_block_from_host() but there may be others.

The following patch may help.
Any chance you could try to get a backtrace from QEMU when it failed?

diff --git a/system/physmem.c b/system/physmem.c
index 33d09f7571..2669c4dbbb 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2277,6 +2277,9 @@ RAMBlock *qemu_ram_block_from_host(void *ptr, bool
round_offset,
 ram_addr_t ram_addr;
 RCU_READ_LOCK_GUARD();
 ram_addr = xen_ram_addr_from_mapcache(ptr);
+if (ram_addr == RAM_ADDR_INVALID) {
+return NULL;
+}
 block = qemu_get_ram_block(ram_addr);
 if (block) {
 *offset = ram_addr - block->offset;





> Cheers,
>
> Anthony
>
> On Wed, May 29, 2024 at 04:07:33PM +0200, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > When invalidating memory ranges, if we happen to hit the first
> > entry in a bucket we were never unmapping it. This was harmless
> > for foreign mappings but now that we're looking to reuse the
> > mapcache for transient grant mappings, we must unmap entries
> > when invalidated.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > Reviewed-by: Stefano Stabellini 
> > ---
> >  hw/xen/xen-mapcache.c | 11 ---
> >  1 file changed, 8 insertions(+), 3 deletions(-)
> >
> > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > index bc860f4373..ec95445696 100644
> > --- a/hw/xen/xen-mapcache.c
> > +++ b/hw/xen/xen-mapcache.c
> > @@ -491,18 +491,23 @@ static void
> xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
> >  return;
> >  }
> >  entry->lock--;
> > -if (entry->lock > 0 || pentry == NULL) {
> > +if (entry->lock > 0) {
> >  return;
> >  }
> >
> > -pentry->next = entry->next;
> >  ram_block_notify_remove(entry->vaddr_base, entry->size,
> entry->size);
> >  if (munmap(entry->vaddr_base, entry->size) != 0) {
> >  perror("unmap fails");
> >  exit(-1);
> >  }
> > +
> >  g_free(entry->valid_mapping);
> > -g_free(entry);
> > +if (pentry) {
> > +pentry->next = entry->next;
> > +g_free(entry);
> > +} else {
> > +memset(entry, 0, sizeof *entry);
> > +}
> >  }
> >
> >  typedef struct XenMapCacheData {
> > --
> > 2.40.1
> >
> >
> --
>
> Anthony Perard | Vates XCP-ng Developer
>
> XCP-ng & Xen Orchestra - Vates solutions
>
> web: https://vates.tech
>

[PATCH v8 1/8] xen: mapcache: Make MCACHE_BUCKET_SHIFT runtime configurable

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Make MCACHE_BUCKET_SHIFT runtime configurable per cache instance.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 54 ++-
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index fa6813b1ad..bc860f4373 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -23,13 +23,10 @@
 
 
 #if HOST_LONG_BITS == 32
-#  define MCACHE_BUCKET_SHIFT 16
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
 #else
-#  define MCACHE_BUCKET_SHIFT 20
 #  define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
 #endif
-#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
 
 /* This is the size of the virtual address space reserve to QEMU that will not
  * be use by MapCache.
@@ -65,7 +62,8 @@ typedef struct MapCache {
 /* For most cases (>99.9%), the page address is the same. */
 MapCacheEntry *last_entry;
 unsigned long max_mcache_size;
-unsigned int mcache_bucket_shift;
+unsigned int bucket_shift;
+unsigned long bucket_size;
 
 phys_offset_to_gaddr_t phys_offset_to_gaddr;
 QemuMutex lock;
@@ -95,11 +93,14 @@ static inline int test_bits(int nr, int size, const 
unsigned long *addr)
 
 static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f,
void *opaque,
+   unsigned int bucket_shift,
unsigned long max_size)
 {
 unsigned long size;
 MapCache *mc;
 
+assert(bucket_shift >= XC_PAGE_SHIFT);
+
 mc = g_new0(MapCache, 1);
 
 mc->phys_offset_to_gaddr = f;
@@ -108,12 +109,14 @@ static MapCache 
*xen_map_cache_init_single(phys_offset_to_gaddr_t f,
 
 QTAILQ_INIT(&mc->locked_entries);
 
+mc->bucket_shift = bucket_shift;
+mc->bucket_size = 1UL << bucket_shift;
 mc->max_mcache_size = max_size;
 
 mc->nr_buckets =
 (((mc->max_mcache_size >> XC_PAGE_SHIFT) +
-  (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
- (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
+  (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >>
+ (bucket_shift - XC_PAGE_SHIFT));
 
 size = mc->nr_buckets * sizeof(MapCacheEntry);
 size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
@@ -126,6 +129,13 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 {
 struct rlimit rlimit_as;
 unsigned long max_mcache_size;
+unsigned int bucket_shift;
+
+if (HOST_LONG_BITS == 32) {
+bucket_shift = 16;
+} else {
+bucket_shift = 20;
+}
 
 if (geteuid() == 0) {
 rlimit_as.rlim_cur = RLIM_INFINITY;
@@ -146,7 +156,9 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 }
 }
 
-mapcache = xen_map_cache_init_single(f, opaque, max_mcache_size);
+mapcache = xen_map_cache_init_single(f, opaque,
+ bucket_shift,
+ max_mcache_size);
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -195,7 +207,7 @@ static void xen_remap_bucket(MapCache *mc,
 entry->valid_mapping = NULL;
 
 for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
 }
 
 /*
@@ -266,8 +278,8 @@ static uint8_t *xen_map_cache_unlocked(MapCache *mc,
 bool dummy = false;
 
 tryagain:
-address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
-address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
+address_index  = phys_addr >> mc->bucket_shift;
+address_offset = phys_addr & (mc->bucket_size - 1);
 
 trace_xen_map_cache(phys_addr);
 
@@ -294,14 +306,14 @@ tryagain:
 return mc->last_entry->vaddr_base + address_offset;
 }
 
-/* size is always a multiple of MCACHE_BUCKET_SIZE */
+/* size is always a multiple of mc->bucket_size */
 if (size) {
 cache_size = size + address_offset;
-if (cache_size % MCACHE_BUCKET_SIZE) {
-cache_size += MCACHE_BUCKET_SIZE - (cache_size % 
MCACHE_BUCKET_SIZE);
+if (cache_size % mc->bucket_size) {
+cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
 }
 } else {
-cache_size = MCACHE_BUCKET_SIZE;
+cache_size = mc->bucket_size;
 }
 
 entry = &mc->entry[address_index % mc->nr_buckets];
@@ -422,7 +434,7 @@ static ram_addr_t 
xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr)
 trace_xen_ram_addr_from_mapcache_not_in_cache(ptr);
 raddr = RAM_ADDR_INVALID;
 } else {
-raddr = (reventry->paddr_index << MCACHE_B

[PATCH v8 7/8] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a second mapcache for grant mappings. The mapcache for
grants needs to work with XC_PAGE_SIZE granularity since
we can't map larger ranges than what has been granted to us.

Like with foreign mappings (xen_memory), machines using grants
are expected to initialize the xen_grants MR and map it
into their address-map accordingly.

CC: Manos Pitsidianakis 
Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-hvm-common.c |  12 ++-
 hw/xen/xen-mapcache.c   | 165 +---
 include/hw/xen/xen-hvm-common.h |   3 +
 include/sysemu/xen.h|   1 +
 4 files changed, 144 insertions(+), 37 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index a0a0252da0..b8ace1c368 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -10,12 +10,18 @@
 #include "hw/boards.h"
 #include "hw/xen/arch_hvm.h"
 
-MemoryRegion xen_memory;
+MemoryRegion xen_memory, xen_grants;
 
-/* Check for xen memory.  */
+/* Check for any kind of xen memory, foreign mappings or grants.  */
 bool xen_mr_is_memory(MemoryRegion *mr)
 {
-return mr == &xen_memory;
+return mr == &xen_memory || mr == &xen_grants;
+}
+
+/* Check specifically for grants.  */
+bool xen_mr_is_grants(MemoryRegion *mr)
+{
+return mr == &xen_grants;
 }
 
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index a07c47b0b1..5f23b0adbe 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -14,6 +14,7 @@
 
 #include 
 
+#include "hw/xen/xen-hvm-common.h"
 #include "hw/xen/xen_native.h"
 #include "qemu/bitmap.h"
 
@@ -21,6 +22,8 @@
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
+#include 
+#include 
 
 #if HOST_LONG_BITS == 32
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
@@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
 unsigned long *valid_mapping;
 uint32_t lock;
 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
+#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
 uint8_t flags;
 hwaddr size;
 struct MapCacheEntry *next;
@@ -71,6 +75,8 @@ typedef struct MapCache {
 } MapCache;
 
 static MapCache *mapcache;
+static MapCache *mapcache_grants;
+static xengnttab_handle *xen_region_gnttabdev;
 
 static inline void mapcache_lock(MapCache *mc)
 {
@@ -131,6 +137,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 unsigned long max_mcache_size;
 unsigned int bucket_shift;
 
+xen_region_gnttabdev = xengnttab_open(NULL, 0);
+if (xen_region_gnttabdev == NULL) {
+error_report("mapcache: Failed to open gnttab device");
+exit(EXIT_FAILURE);
+}
+
 if (HOST_LONG_BITS == 32) {
 bucket_shift = 16;
 } else {
@@ -159,6 +171,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 mapcache = xen_map_cache_init_single(f, opaque,
  bucket_shift,
  max_mcache_size);
+
+/*
+ * Grant mappings must use XC_PAGE_SIZE granularity since we can't
+ * map anything beyond the number of pages granted to us.
+ */
+mapcache_grants = xen_map_cache_init_single(f, opaque,
+XC_PAGE_SHIFT,
+max_mcache_size);
+
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -168,17 +189,24 @@ static void xen_remap_bucket(MapCache *mc,
  hwaddr size,
  hwaddr address_index,
  bool dummy,
+ bool grant,
+ bool is_write,
  ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
-xen_pfn_t *pfns;
-int *err;
+g_autofree uint32_t *refs = NULL;
+g_autofree xen_pfn_t *pfns = NULL;
+g_autofree int *err;
 unsigned int i;
 hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
 trace_xen_remap_bucket(address_index);
 
-pfns = g_new0(xen_pfn_t, nb_pfn);
+if (grant) {
+refs = g_new0(uint32_t, nb_pfn);
+} else {
+pfns = g_new0(xen_pfn_t, nb_pfn);
+}
 err = g_new0(int, nb_pfn);
 
 if (entry->vaddr_base != NULL) {
@@ -207,21 +235,51 @@ static void xen_remap_bucket(MapCache *mc,
 g_free(entry->valid_mapping);
 entry->valid_mapping = NULL;
 
-for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
+if (grant) {
+hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT);
+
+for (i = 0; i < nb_pfn; i++) {
+refs[i] = grant_base + i;
+}
+} else {
+for (i = 0; i < nb_pfn; i++) {
+  

[PATCH v8 3/8] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add xen_mr_is_memory() to abstract away tests for the
xen_memory MR.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
Acked-by: David Hildenbrand 
Reviewed-by: Philippe Mathieu-Daudé 
---
 hw/xen/xen-hvm-common.c | 10 --
 include/sysemu/xen.h|  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index 2d1b032121..a0a0252da0 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -12,6 +12,12 @@
 
 MemoryRegion xen_memory;
 
+/* Check for xen memory.  */
+bool xen_mr_is_memory(MemoryRegion *mr)
+{
+return mr == &xen_memory;
+}
+
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
Error **errp)
 {
@@ -28,7 +34,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, 
MemoryRegion *mr,
 return;
 }
 
-if (mr == &xen_memory) {
+if (xen_mr_is_memory(mr)) {
 return;
 }
 
@@ -55,7 +61,7 @@ static void xen_set_memory(struct MemoryListener *listener,
 {
 XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-if (section->mr == &xen_memory) {
+if (xen_mr_is_memory(section->mr)) {
 return;
 } else {
 if (add) {
diff --git a/include/sysemu/xen.h b/include/sysemu/xen.h
index 754ec2e6cb..3445888e39 100644
--- a/include/sysemu/xen.h
+++ b/include/sysemu/xen.h
@@ -49,4 +49,5 @@ static inline void xen_ram_alloc(ram_addr_t ram_addr, 
ram_addr_t size,
 
 #endif /* CONFIG_XEN_IS_POSSIBLE */
 
+bool xen_mr_is_memory(MemoryRegion *mr);
 #endif
-- 
2.40.1

[PATCH v8 6/8] xen: mapcache: Pass the ram_addr offset to xen_map_cache()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Pass the ram_addr offset to xen_map_cache.
This is in preparation for adding grant mappings that need
to compute the address within the RAMBlock.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: David Hildenbrand 
Reviewed-by: Stefano Stabellini 
Reviewed-by: Philippe Mathieu-Daudé 
---
 hw/xen/xen-mapcache.c | 16 +++-
 include/sysemu/xen-mapcache.h |  2 ++
 system/physmem.c  |  9 +
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index ec95445696..a07c47b0b1 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -167,7 +167,8 @@ static void xen_remap_bucket(MapCache *mc,
  void *vaddr,
  hwaddr size,
  hwaddr address_index,
- bool dummy)
+ bool dummy,
+ ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
 xen_pfn_t *pfns;
@@ -266,6 +267,7 @@ static void xen_remap_bucket(MapCache *mc,
 
 static uint8_t *xen_map_cache_unlocked(MapCache *mc,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_offset,
uint8_t lock, bool dma, bool is_write)
 {
 MapCacheEntry *entry, *pentry = NULL,
@@ -337,14 +339,16 @@ tryagain:
 if (!entry) {
 entry = g_new0(MapCacheEntry, 1);
 pentry->next = entry;
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 } else if (!entry->lock) {
 if (!entry->vaddr_base || entry->paddr_index != address_index ||
 entry->size != cache_size ||
 !test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 }
 }
 
@@ -391,13 +395,15 @@ tryagain:
 
 uint8_t *xen_map_cache(MemoryRegion *mr,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write)
 {
 uint8_t *p;
 
 mapcache_lock(mapcache);
-p = xen_map_cache_unlocked(mapcache, phys_addr, size, lock, dma, is_write);
+p = xen_map_cache_unlocked(mapcache, phys_addr, size, ram_addr_offset,
+   lock, dma, is_write);
 mapcache_unlock(mapcache);
 return p;
 }
@@ -632,7 +638,7 @@ static uint8_t *xen_replace_cache_entry_unlocked(MapCache 
*mc,
 trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
 
 xen_remap_bucket(mc, entry, entry->vaddr_base,
- cache_size, address_index, false);
+ cache_size, address_index, false, old_phys_addr);
 if (!test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index 1ec9e66752..b5e3ea1bc0 100644
--- a/include/sysemu/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -19,6 +19,7 @@ typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr phys_offset,
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
 void *opaque);
 uint8_t *xen_map_cache(MemoryRegion *mr, hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
@@ -37,6 +38,7 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t 
f,
 static inline uint8_t *xen_map_cache(MemoryRegion *mr,
  hwaddr phys_addr,
  hwaddr size,
+ ram_addr_t ram_addr_offset,
  uint8_t lock,
  bool dma,
  bool is_write)
diff --git a/system/physmem.c b/system/physmem.c
index b7847db1a2..33d09f7571 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2231,13 +2231,14 @@ static void *qemu_ram_ptr_length(RAMBlock *block, 
ram_addr_t addr,
  */
 if (xen_mr_is_memory(block->mr)) {
 return xen_map_cache(block->mr, block->offset + addr,
- len, lock, lock,
- is_write);
+  

[PATCH v8 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

When invalidating memory ranges, if we happen to hit the first
entry in a bucket we were never unmapping it. This was harmless
for foreign mappings but now that we're looking to reuse the
mapcache for transient grant mappings, we must unmap entries
when invalidated.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index bc860f4373..ec95445696 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -491,18 +491,23 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 return;
 }
 entry->lock--;
-if (entry->lock > 0 || pentry == NULL) {
+if (entry->lock > 0) {
 return;
 }
 
-pentry->next = entry->next;
 ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
 if (munmap(entry->vaddr_base, entry->size) != 0) {
 perror("unmap fails");
 exit(-1);
 }
+
 g_free(entry->valid_mapping);
-g_free(entry);
+if (pentry) {
+pentry->next = entry->next;
+g_free(entry);
+} else {
+memset(entry, 0, sizeof *entry);
+}
 }
 
 typedef struct XenMapCacheData {
-- 
2.40.1

Re: [PATCH v7 3/8] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

On Mon, May 27, 2024 at 6:25 PM Philippe Mathieu-Daudé 
wrote:

> Hi Edgar,
>
> On 24/5/24 12:51, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > Add xen_mr_is_memory() to abstract away tests for the
> > xen_memory MR.
> >
> > No functional changes.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > Reviewed-by: Stefano Stabellini 
> > Acked-by: David Hildenbrand 
> > ---
> >   hw/xen/xen-hvm-common.c | 10 --
> >   include/sysemu/xen.h|  8 
> >   2 files changed, 16 insertions(+), 2 deletions(-)
>
> To consolidate we could add:
>
>static MemoryRegion xen_memory;
>
>MemoryRegion *xen_mr_memory_init(uint64_t block_len)
>{
>   assert(!xen_memory.size);
>   memory_region_init_ram(&xen_memory, NULL, "xen.ram", block_len,
> &error_fatal);
>   return &xen_memory;
>}
>
> and remove the extern declaration.
>

Thanks,

We have a future patch series in the workings that will add a PVH machine
for x86, I'll keep this in mind for then!


>
> > diff --git a/include/sysemu/xen.h b/include/sysemu/xen.h
> > index 754ec2e6cb..dc72f83bcb 100644
> > --- a/include/sysemu/xen.h
> > +++ b/include/sysemu/xen.h
> > @@ -34,6 +34,8 @@ void xen_hvm_modified_memory(ram_addr_t start,
> ram_addr_t length);
> >   void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
> >  struct MemoryRegion *mr, Error **errp);
> >
> > +bool xen_mr_is_memory(MemoryRegion *mr);
> > +
> >   #else /* !CONFIG_XEN_IS_POSSIBLE */
> >
> >   #define xen_enabled() 0
> > @@ -47,6 +49,12 @@ static inline void xen_ram_alloc(ram_addr_t ram_addr,
> ram_addr_t size,
> >   g_assert_not_reached();
> >   }
> >
> > +static inline bool xen_mr_is_memory(MemoryRegion *mr)
> > +{
> > +g_assert_not_reached();
> > +return false;
>
> No need for the stub, just always declare xen_mr_is_memory() ...
> > +}
> > +
> >   #endif /* CONFIG_XEN_IS_POSSIBLE */
>
> ... here.
>
> >   #endif
>
> Removing the stub:
> Reviewed-by: Philippe Mathieu-Daudé 
>
>
Thanks Philippe, will remove the stubs in v8.

Cheers,
Edgar

[PATCH v7 6/8] xen: mapcache: Pass the ram_addr offset to xen_map_cache()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Pass the ram_addr offset to xen_map_cache.
This is in preparation for adding grant mappings that need
to compute the address within the RAMBlock.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: David Hildenbrand 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 16 +++-
 include/sysemu/xen-mapcache.h |  2 ++
 system/physmem.c  |  9 +
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index ec95445696..a07c47b0b1 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -167,7 +167,8 @@ static void xen_remap_bucket(MapCache *mc,
  void *vaddr,
  hwaddr size,
  hwaddr address_index,
- bool dummy)
+ bool dummy,
+ ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
 xen_pfn_t *pfns;
@@ -266,6 +267,7 @@ static void xen_remap_bucket(MapCache *mc,
 
 static uint8_t *xen_map_cache_unlocked(MapCache *mc,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_offset,
uint8_t lock, bool dma, bool is_write)
 {
 MapCacheEntry *entry, *pentry = NULL,
@@ -337,14 +339,16 @@ tryagain:
 if (!entry) {
 entry = g_new0(MapCacheEntry, 1);
 pentry->next = entry;
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 } else if (!entry->lock) {
 if (!entry->vaddr_base || entry->paddr_index != address_index ||
 entry->size != cache_size ||
 !test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 }
 }
 
@@ -391,13 +395,15 @@ tryagain:
 
 uint8_t *xen_map_cache(MemoryRegion *mr,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write)
 {
 uint8_t *p;
 
 mapcache_lock(mapcache);
-p = xen_map_cache_unlocked(mapcache, phys_addr, size, lock, dma, is_write);
+p = xen_map_cache_unlocked(mapcache, phys_addr, size, ram_addr_offset,
+   lock, dma, is_write);
 mapcache_unlock(mapcache);
 return p;
 }
@@ -632,7 +638,7 @@ static uint8_t *xen_replace_cache_entry_unlocked(MapCache 
*mc,
 trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
 
 xen_remap_bucket(mc, entry, entry->vaddr_base,
- cache_size, address_index, false);
+ cache_size, address_index, false, old_phys_addr);
 if (!test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index 1ec9e66752..b5e3ea1bc0 100644
--- a/include/sysemu/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -19,6 +19,7 @@ typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr phys_offset,
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
 void *opaque);
 uint8_t *xen_map_cache(MemoryRegion *mr, hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
@@ -37,6 +38,7 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t 
f,
 static inline uint8_t *xen_map_cache(MemoryRegion *mr,
  hwaddr phys_addr,
  hwaddr size,
+ ram_addr_t ram_addr_offset,
  uint8_t lock,
  bool dma,
  bool is_write)
diff --git a/system/physmem.c b/system/physmem.c
index b7847db1a2..33d09f7571 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2231,13 +2231,14 @@ static void *qemu_ram_ptr_length(RAMBlock *block, 
ram_addr_t addr,
  */
 if (xen_mr_is_memory(block->mr)) {
 return xen_map_cache(block->mr, block->offset + addr,
- len, lock, lock,
- is_write);
+ len, block->offset,
+ 

[PATCH v7 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

When invalidating memory ranges, if we happen to hit the first
entry in a bucket we were never unmapping it. This was harmless
for foreign mappings but now that we're looking to reuse the
mapcache for transient grant mappings, we must unmap entries
when invalidated.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index bc860f4373..ec95445696 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -491,18 +491,23 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 return;
 }
 entry->lock--;
-if (entry->lock > 0 || pentry == NULL) {
+if (entry->lock > 0) {
 return;
 }
 
-pentry->next = entry->next;
 ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
 if (munmap(entry->vaddr_base, entry->size) != 0) {
 perror("unmap fails");
 exit(-1);
 }
+
 g_free(entry->valid_mapping);
-g_free(entry);
+if (pentry) {
+pentry->next = entry->next;
+g_free(entry);
+} else {
+memset(entry, 0, sizeof *entry);
+}
 }
 
 typedef struct XenMapCacheData {
-- 
2.40.1

[PATCH v7 7/8] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a second mapcache for grant mappings. The mapcache for
grants needs to work with XC_PAGE_SIZE granularity since
we can't map larger ranges than what has been granted to us.

Like with foreign mappings (xen_memory), machines using grants
are expected to initialize the xen_grants MR and map it
into their address-map accordingly.

CC: Manos Pitsidianakis 
Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-hvm-common.c |  12 ++-
 hw/xen/xen-mapcache.c   | 165 +---
 include/hw/xen/xen-hvm-common.h |   3 +
 include/sysemu/xen.h|   7 ++
 4 files changed, 150 insertions(+), 37 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index a0a0252da0..b8ace1c368 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -10,12 +10,18 @@
 #include "hw/boards.h"
 #include "hw/xen/arch_hvm.h"
 
-MemoryRegion xen_memory;
+MemoryRegion xen_memory, xen_grants;
 
-/* Check for xen memory.  */
+/* Check for any kind of xen memory, foreign mappings or grants.  */
 bool xen_mr_is_memory(MemoryRegion *mr)
 {
-return mr == &xen_memory;
+return mr == &xen_memory || mr == &xen_grants;
+}
+
+/* Check specifically for grants.  */
+bool xen_mr_is_grants(MemoryRegion *mr)
+{
+return mr == &xen_grants;
 }
 
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index a07c47b0b1..5f23b0adbe 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -14,6 +14,7 @@
 
 #include 
 
+#include "hw/xen/xen-hvm-common.h"
 #include "hw/xen/xen_native.h"
 #include "qemu/bitmap.h"
 
@@ -21,6 +22,8 @@
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
+#include 
+#include 
 
 #if HOST_LONG_BITS == 32
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
@@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
 unsigned long *valid_mapping;
 uint32_t lock;
 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
+#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
 uint8_t flags;
 hwaddr size;
 struct MapCacheEntry *next;
@@ -71,6 +75,8 @@ typedef struct MapCache {
 } MapCache;
 
 static MapCache *mapcache;
+static MapCache *mapcache_grants;
+static xengnttab_handle *xen_region_gnttabdev;
 
 static inline void mapcache_lock(MapCache *mc)
 {
@@ -131,6 +137,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 unsigned long max_mcache_size;
 unsigned int bucket_shift;
 
+xen_region_gnttabdev = xengnttab_open(NULL, 0);
+if (xen_region_gnttabdev == NULL) {
+error_report("mapcache: Failed to open gnttab device");
+exit(EXIT_FAILURE);
+}
+
 if (HOST_LONG_BITS == 32) {
 bucket_shift = 16;
 } else {
@@ -159,6 +171,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 mapcache = xen_map_cache_init_single(f, opaque,
  bucket_shift,
  max_mcache_size);
+
+/*
+ * Grant mappings must use XC_PAGE_SIZE granularity since we can't
+ * map anything beyond the number of pages granted to us.
+ */
+mapcache_grants = xen_map_cache_init_single(f, opaque,
+XC_PAGE_SHIFT,
+max_mcache_size);
+
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -168,17 +189,24 @@ static void xen_remap_bucket(MapCache *mc,
  hwaddr size,
  hwaddr address_index,
  bool dummy,
+ bool grant,
+ bool is_write,
  ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
-xen_pfn_t *pfns;
-int *err;
+g_autofree uint32_t *refs = NULL;
+g_autofree xen_pfn_t *pfns = NULL;
+g_autofree int *err;
 unsigned int i;
 hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
 trace_xen_remap_bucket(address_index);
 
-pfns = g_new0(xen_pfn_t, nb_pfn);
+if (grant) {
+refs = g_new0(uint32_t, nb_pfn);
+} else {
+pfns = g_new0(xen_pfn_t, nb_pfn);
+}
 err = g_new0(int, nb_pfn);
 
 if (entry->vaddr_base != NULL) {
@@ -207,21 +235,51 @@ static void xen_remap_bucket(MapCache *mc,
 g_free(entry->valid_mapping);
 entry->valid_mapping = NULL;
 
-for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
+if (grant) {
+hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT);
+
+for (i = 0; i < nb_pfn; i++) {
+refs[i] = grant_base + i;
+}
+} else {
+for (i = 0; i < nb_pfn; i++) {
+  

[PATCH v7 3/8] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add xen_mr_is_memory() to abstract away tests for the
xen_memory MR.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
Acked-by: David Hildenbrand 
---
 hw/xen/xen-hvm-common.c | 10 --
 include/sysemu/xen.h|  8 
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index 2d1b032121..a0a0252da0 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -12,6 +12,12 @@
 
 MemoryRegion xen_memory;
 
+/* Check for xen memory.  */
+bool xen_mr_is_memory(MemoryRegion *mr)
+{
+return mr == &xen_memory;
+}
+
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
Error **errp)
 {
@@ -28,7 +34,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, 
MemoryRegion *mr,
 return;
 }
 
-if (mr == &xen_memory) {
+if (xen_mr_is_memory(mr)) {
 return;
 }
 
@@ -55,7 +61,7 @@ static void xen_set_memory(struct MemoryListener *listener,
 {
 XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-if (section->mr == &xen_memory) {
+if (xen_mr_is_memory(section->mr)) {
 return;
 } else {
 if (add) {
diff --git a/include/sysemu/xen.h b/include/sysemu/xen.h
index 754ec2e6cb..dc72f83bcb 100644
--- a/include/sysemu/xen.h
+++ b/include/sysemu/xen.h
@@ -34,6 +34,8 @@ void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t 
length);
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
struct MemoryRegion *mr, Error **errp);
 
+bool xen_mr_is_memory(MemoryRegion *mr);
+
 #else /* !CONFIG_XEN_IS_POSSIBLE */
 
 #define xen_enabled() 0
@@ -47,6 +49,12 @@ static inline void xen_ram_alloc(ram_addr_t ram_addr, 
ram_addr_t size,
 g_assert_not_reached();
 }
 
+static inline bool xen_mr_is_memory(MemoryRegion *mr)
+{
+g_assert_not_reached();
+return false;
+}
+
 #endif /* CONFIG_XEN_IS_POSSIBLE */
 
 #endif
-- 
2.40.1

[PATCH v7 1/8] xen: mapcache: Make MCACHE_BUCKET_SHIFT runtime configurable

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Make MCACHE_BUCKET_SHIFT runtime configurable per cache instance.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 54 ++-
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index fa6813b1ad..bc860f4373 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -23,13 +23,10 @@
 
 
 #if HOST_LONG_BITS == 32
-#  define MCACHE_BUCKET_SHIFT 16
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
 #else
-#  define MCACHE_BUCKET_SHIFT 20
 #  define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
 #endif
-#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
 
 /* This is the size of the virtual address space reserve to QEMU that will not
  * be use by MapCache.
@@ -65,7 +62,8 @@ typedef struct MapCache {
 /* For most cases (>99.9%), the page address is the same. */
 MapCacheEntry *last_entry;
 unsigned long max_mcache_size;
-unsigned int mcache_bucket_shift;
+unsigned int bucket_shift;
+unsigned long bucket_size;
 
 phys_offset_to_gaddr_t phys_offset_to_gaddr;
 QemuMutex lock;
@@ -95,11 +93,14 @@ static inline int test_bits(int nr, int size, const 
unsigned long *addr)
 
 static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f,
void *opaque,
+   unsigned int bucket_shift,
unsigned long max_size)
 {
 unsigned long size;
 MapCache *mc;
 
+assert(bucket_shift >= XC_PAGE_SHIFT);
+
 mc = g_new0(MapCache, 1);
 
 mc->phys_offset_to_gaddr = f;
@@ -108,12 +109,14 @@ static MapCache 
*xen_map_cache_init_single(phys_offset_to_gaddr_t f,
 
 QTAILQ_INIT(&mc->locked_entries);
 
+mc->bucket_shift = bucket_shift;
+mc->bucket_size = 1UL << bucket_shift;
 mc->max_mcache_size = max_size;
 
 mc->nr_buckets =
 (((mc->max_mcache_size >> XC_PAGE_SHIFT) +
-  (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
- (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
+  (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >>
+ (bucket_shift - XC_PAGE_SHIFT));
 
 size = mc->nr_buckets * sizeof(MapCacheEntry);
 size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
@@ -126,6 +129,13 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 {
 struct rlimit rlimit_as;
 unsigned long max_mcache_size;
+unsigned int bucket_shift;
+
+if (HOST_LONG_BITS == 32) {
+bucket_shift = 16;
+} else {
+bucket_shift = 20;
+}
 
 if (geteuid() == 0) {
 rlimit_as.rlim_cur = RLIM_INFINITY;
@@ -146,7 +156,9 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 }
 }
 
-mapcache = xen_map_cache_init_single(f, opaque, max_mcache_size);
+mapcache = xen_map_cache_init_single(f, opaque,
+ bucket_shift,
+ max_mcache_size);
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -195,7 +207,7 @@ static void xen_remap_bucket(MapCache *mc,
 entry->valid_mapping = NULL;
 
 for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
 }
 
 /*
@@ -266,8 +278,8 @@ static uint8_t *xen_map_cache_unlocked(MapCache *mc,
 bool dummy = false;
 
 tryagain:
-address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
-address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
+address_index  = phys_addr >> mc->bucket_shift;
+address_offset = phys_addr & (mc->bucket_size - 1);
 
 trace_xen_map_cache(phys_addr);
 
@@ -294,14 +306,14 @@ tryagain:
 return mc->last_entry->vaddr_base + address_offset;
 }
 
-/* size is always a multiple of MCACHE_BUCKET_SIZE */
+/* size is always a multiple of mc->bucket_size */
 if (size) {
 cache_size = size + address_offset;
-if (cache_size % MCACHE_BUCKET_SIZE) {
-cache_size += MCACHE_BUCKET_SIZE - (cache_size % 
MCACHE_BUCKET_SIZE);
+if (cache_size % mc->bucket_size) {
+cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
 }
 } else {
-cache_size = MCACHE_BUCKET_SIZE;
+cache_size = mc->bucket_size;
 }
 
 entry = &mc->entry[address_index % mc->nr_buckets];
@@ -422,7 +434,7 @@ static ram_addr_t 
xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr)
 trace_xen_ram_addr_from_mapcache_not_in_cache(ptr);
 raddr = RAM_ADDR_INVALID;
 } else {
-raddr = (reventry->paddr_index << MCACHE_B

[PATCH v1 0/1] xen/arm: smmuv3: Mark more init-only functions with __init

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

I was scanning for code that we could potentially move from the
.text section into .init.text and found a few candidates.

I'm not sure if this makes sense, perhaps we don't want to mark
these functions for other reasons but my scripts found this chain
of SMMUv3 init functions as only reachable by .inittext code.
Perhaps it's a little late in the release cycle to consider this...

Best regards,
Edgar


Edgar E. Iglesias (1):
  xen/arm: smmuv3: Mark more init-only functions with __init

 xen/drivers/passthrough/arm/smmu-v3.c | 17 +
 1 file changed, 9 insertions(+), 8 deletions(-)

-- 
2.40.1

[PATCH v1 1/1] xen/arm: smmuv3: Mark more init-only functions with __init

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Move more functions that are only called at init to
the .init.text section.

Signed-off-by: Edgar E. Iglesias 
---
 xen/drivers/passthrough/arm/smmu-v3.c | 17 +
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/xen/drivers/passthrough/arm/smmu-v3.c 
b/xen/drivers/passthrough/arm/smmu-v3.c
index 6904962467..cee5724022 100644
--- a/xen/drivers/passthrough/arm/smmu-v3.c
+++ b/xen/drivers/passthrough/arm/smmu-v3.c
@@ -1545,7 +1545,7 @@ static int arm_smmu_dt_xlate(struct device *dev,
 }
 
 /* Probing and initialisation functions */
-static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
+static int __init arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
   struct arm_smmu_queue *q,
   void __iomem *page,
   unsigned long prod_off,
@@ -1588,7 +1588,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device 
*smmu,
return 0;
 }
 
-static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
+static int __init arm_smmu_init_queues(struct arm_smmu_device *smmu)
 {
int ret;
 
@@ -1724,7 +1724,7 @@ static int arm_smmu_init_strtab(struct arm_smmu_device 
*smmu)
return 0;
 }
 
-static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
+static int __init arm_smmu_init_structures(struct arm_smmu_device *smmu)
 {
int ret;
 
@@ -1746,7 +1746,8 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device 
*smmu, u32 val,
 }
 
 /* GBPA is "special" */
-static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
+static int __init arm_smmu_update_gbpa(struct arm_smmu_device *smmu,
+   u32 set, u32 clr)
 {
int ret;
u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
@@ -1842,7 +1843,7 @@ static void arm_smmu_setup_msis(struct arm_smmu_device 
*smmu)
 static inline void arm_smmu_setup_msis(struct arm_smmu_device *smmu) { }
 #endif /* CONFIG_MSI */
 
-static void arm_smmu_free_irqs(struct arm_smmu_device *smmu)
+static void __init arm_smmu_free_irqs(struct arm_smmu_device *smmu)
 {
int irq;
 
@@ -1926,7 +1927,7 @@ err_free_evtq_irq:
return ret;
 }
 
-static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
+static int __init arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
 {
int ret, irq;
u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
@@ -1988,7 +1989,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device 
*smmu)
return ret;
 }
 
-static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
+static int __init arm_smmu_device_reset(struct arm_smmu_device *smmu)
 {
int ret;
u32 reg, enables;
@@ -2405,7 +2406,7 @@ static void arm_smmu_free_structures(struct 
arm_smmu_device *smmu)
xfree(smmu->strtab_cfg.l1_desc);
 }
 
-static int arm_smmu_device_probe(struct platform_device *pdev)
+static int __init arm_smmu_device_probe(struct platform_device *pdev)
 {
int irq, ret;
paddr_t ioaddr, iosize;
-- 
2.40.1

Re: [PATCH v6 7/8] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

On Thu, May 23, 2024 at 9:47 AM Manos Pitsidianakis <
manos.pitsidiana...@linaro.org> wrote:

> On Thu, 16 May 2024 18:48, "Edgar E. Iglesias" 
> wrote:
> >From: "Edgar E. Iglesias" 
> >
> >Add a second mapcache for grant mappings. The mapcache for
> >grants needs to work with XC_PAGE_SIZE granularity since
> >we can't map larger ranges than what has been granted to us.
> >
> >Like with foreign mappings (xen_memory), machines using grants
> >are expected to initialize the xen_grants MR and map it
> >into their address-map accordingly.
> >
> >Signed-off-by: Edgar E. Iglesias 
> >Reviewed-by: Stefano Stabellini 
> >---
> > hw/xen/xen-hvm-common.c |  12 ++-
> > hw/xen/xen-mapcache.c   | 163 ++--
> > include/hw/xen/xen-hvm-common.h |   3 +
> > include/sysemu/xen.h|   7 ++
> > 4 files changed, 152 insertions(+), 33 deletions(-)
> >
> >diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
> >index a0a0252da0..b8ace1c368 100644
> >--- a/hw/xen/xen-hvm-common.c
> >+++ b/hw/xen/xen-hvm-common.c
> >@@ -10,12 +10,18 @@
> > #include "hw/boards.h"
> > #include "hw/xen/arch_hvm.h"
> >
> >-MemoryRegion xen_memory;
> >+MemoryRegion xen_memory, xen_grants;
> >
> >-/* Check for xen memory.  */
> >+/* Check for any kind of xen memory, foreign mappings or grants.  */
> > bool xen_mr_is_memory(MemoryRegion *mr)
> > {
> >-return mr == &xen_memory;
> >+return mr == &xen_memory || mr == &xen_grants;
> >+}
> >+
> >+/* Check specifically for grants.  */
> >+bool xen_mr_is_grants(MemoryRegion *mr)
> >+{
> >+return mr == &xen_grants;
> > }
> >
> > void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion
> *mr,
> >diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> >index a07c47b0b1..1cbc2aeaa9 100644
> >--- a/hw/xen/xen-mapcache.c
> >+++ b/hw/xen/xen-mapcache.c
> >@@ -14,6 +14,7 @@
> >
> > #include 
> >
> >+#include "hw/xen/xen-hvm-common.h"
> > #include "hw/xen/xen_native.h"
> > #include "qemu/bitmap.h"
> >
> >@@ -21,6 +22,8 @@
> > #include "sysemu/xen-mapcache.h"
> > #include "trace.h"
> >
> >+#include 
> >+#include 
> >
> > #if HOST_LONG_BITS == 32
> > #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
> >@@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
> > unsigned long *valid_mapping;
> > uint32_t lock;
> > #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
> >+#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
>
> Might we get more entry kinds in the future? (for example foreign maps).
> Maybe this could be an enum.
>
>
Perhaps. Foreign mappings are already supported, this flag separates
ordinary foreign mappings from grant foreign mappings.
IMO, since this is not an external interface it's probably better to change
it once we have a concrete use-case at hand.



> > uint8_t flags;
> > hwaddr size;
> > struct MapCacheEntry *next;
> >@@ -71,6 +75,8 @@ typedef struct MapCache {
> > } MapCache;
> >
> > static MapCache *mapcache;
> >+static MapCache *mapcache_grants;
> >+static xengnttab_handle *xen_region_gnttabdev;
> >
> > static inline void mapcache_lock(MapCache *mc)
> > {
> >@@ -131,6 +137,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f,
> void *opaque)
> > unsigned long max_mcache_size;
> > unsigned int bucket_shift;
> >
> >+xen_region_gnttabdev = xengnttab_open(NULL, 0);
> >+if (xen_region_gnttabdev == NULL) {
> >+error_report("mapcache: Failed to open gnttab device");
> >+exit(EXIT_FAILURE);
> >+}
> >+
> > if (HOST_LONG_BITS == 32) {
> > bucket_shift = 16;
> > } else {
> >@@ -159,6 +171,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f,
> void *opaque)
> > mapcache = xen_map_cache_init_single(f, opaque,
> >  bucket_shift,
> >  max_mcache_size);
> >+
> >+/*
> >+ * Grant mappings must use XC_PAGE_SIZE granularity since we can't
> >+ * map anything beyond the number of pages granted to us.
> >+ */
> >+mapcache_grants = xen_map_cache_init_single(f, opaque,
> >+XC_PAGE_SHIFT,
> >+ 

[PATCH v6 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

When invalidating memory ranges, if we happen to hit the first
entry in a bucket we were never unmapping it. This was harmless
for foreign mappings but now that we're looking to reuse the
mapcache for transient grant mappings, we must unmap entries
when invalidated.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index bc860f4373..ec95445696 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -491,18 +491,23 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 return;
 }
 entry->lock--;
-if (entry->lock > 0 || pentry == NULL) {
+if (entry->lock > 0) {
 return;
 }
 
-pentry->next = entry->next;
 ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
 if (munmap(entry->vaddr_base, entry->size) != 0) {
 perror("unmap fails");
 exit(-1);
 }
+
 g_free(entry->valid_mapping);
-g_free(entry);
+if (pentry) {
+pentry->next = entry->next;
+g_free(entry);
+} else {
+memset(entry, 0, sizeof *entry);
+}
 }
 
 typedef struct XenMapCacheData {
-- 
2.40.1

[PATCH v6 6/8] xen: mapcache: Pass the ram_addr offset to xen_map_cache()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Pass the ram_addr offset to xen_map_cache.
This is in preparation for adding grant mappings that need
to compute the address within the RAMBlock.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-mapcache.c | 16 +++-
 include/sysemu/xen-mapcache.h |  2 ++
 system/physmem.c  |  9 +
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index ec95445696..a07c47b0b1 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -167,7 +167,8 @@ static void xen_remap_bucket(MapCache *mc,
  void *vaddr,
  hwaddr size,
  hwaddr address_index,
- bool dummy)
+ bool dummy,
+ ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
 xen_pfn_t *pfns;
@@ -266,6 +267,7 @@ static void xen_remap_bucket(MapCache *mc,
 
 static uint8_t *xen_map_cache_unlocked(MapCache *mc,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_offset,
uint8_t lock, bool dma, bool is_write)
 {
 MapCacheEntry *entry, *pentry = NULL,
@@ -337,14 +339,16 @@ tryagain:
 if (!entry) {
 entry = g_new0(MapCacheEntry, 1);
 pentry->next = entry;
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 } else if (!entry->lock) {
 if (!entry->vaddr_base || entry->paddr_index != address_index ||
 entry->size != cache_size ||
 !test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 }
 }
 
@@ -391,13 +395,15 @@ tryagain:
 
 uint8_t *xen_map_cache(MemoryRegion *mr,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write)
 {
 uint8_t *p;
 
 mapcache_lock(mapcache);
-p = xen_map_cache_unlocked(mapcache, phys_addr, size, lock, dma, is_write);
+p = xen_map_cache_unlocked(mapcache, phys_addr, size, ram_addr_offset,
+   lock, dma, is_write);
 mapcache_unlock(mapcache);
 return p;
 }
@@ -632,7 +638,7 @@ static uint8_t *xen_replace_cache_entry_unlocked(MapCache 
*mc,
 trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
 
 xen_remap_bucket(mc, entry, entry->vaddr_base,
- cache_size, address_index, false);
+ cache_size, address_index, false, old_phys_addr);
 if (!test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index 1ec9e66752..b5e3ea1bc0 100644
--- a/include/sysemu/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -19,6 +19,7 @@ typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr phys_offset,
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
 void *opaque);
 uint8_t *xen_map_cache(MemoryRegion *mr, hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
@@ -37,6 +38,7 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t 
f,
 static inline uint8_t *xen_map_cache(MemoryRegion *mr,
  hwaddr phys_addr,
  hwaddr size,
+ ram_addr_t ram_addr_offset,
  uint8_t lock,
  bool dma,
  bool is_write)
diff --git a/system/physmem.c b/system/physmem.c
index b7847db1a2..33d09f7571 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2231,13 +2231,14 @@ static void *qemu_ram_ptr_length(RAMBlock *block, 
ram_addr_t addr,
  */
 if (xen_mr_is_memory(block->mr)) {
 return xen_map_cache(block->mr, block->offset + addr,
- len, lock, lock,
- is_write);
+ len, block->offset,
+ lock, lock, is_write);

[PATCH v6 7/8] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a second mapcache for grant mappings. The mapcache for
grants needs to work with XC_PAGE_SIZE granularity since
we can't map larger ranges than what has been granted to us.

Like with foreign mappings (xen_memory), machines using grants
are expected to initialize the xen_grants MR and map it
into their address-map accordingly.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-hvm-common.c |  12 ++-
 hw/xen/xen-mapcache.c   | 163 ++--
 include/hw/xen/xen-hvm-common.h |   3 +
 include/sysemu/xen.h|   7 ++
 4 files changed, 152 insertions(+), 33 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index a0a0252da0..b8ace1c368 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -10,12 +10,18 @@
 #include "hw/boards.h"
 #include "hw/xen/arch_hvm.h"
 
-MemoryRegion xen_memory;
+MemoryRegion xen_memory, xen_grants;
 
-/* Check for xen memory.  */
+/* Check for any kind of xen memory, foreign mappings or grants.  */
 bool xen_mr_is_memory(MemoryRegion *mr)
 {
-return mr == &xen_memory;
+return mr == &xen_memory || mr == &xen_grants;
+}
+
+/* Check specifically for grants.  */
+bool xen_mr_is_grants(MemoryRegion *mr)
+{
+return mr == &xen_grants;
 }
 
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index a07c47b0b1..1cbc2aeaa9 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -14,6 +14,7 @@
 
 #include 
 
+#include "hw/xen/xen-hvm-common.h"
 #include "hw/xen/xen_native.h"
 #include "qemu/bitmap.h"
 
@@ -21,6 +22,8 @@
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
+#include 
+#include 
 
 #if HOST_LONG_BITS == 32
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
@@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
 unsigned long *valid_mapping;
 uint32_t lock;
 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
+#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
 uint8_t flags;
 hwaddr size;
 struct MapCacheEntry *next;
@@ -71,6 +75,8 @@ typedef struct MapCache {
 } MapCache;
 
 static MapCache *mapcache;
+static MapCache *mapcache_grants;
+static xengnttab_handle *xen_region_gnttabdev;
 
 static inline void mapcache_lock(MapCache *mc)
 {
@@ -131,6 +137,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 unsigned long max_mcache_size;
 unsigned int bucket_shift;
 
+xen_region_gnttabdev = xengnttab_open(NULL, 0);
+if (xen_region_gnttabdev == NULL) {
+error_report("mapcache: Failed to open gnttab device");
+exit(EXIT_FAILURE);
+}
+
 if (HOST_LONG_BITS == 32) {
 bucket_shift = 16;
 } else {
@@ -159,6 +171,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 mapcache = xen_map_cache_init_single(f, opaque,
  bucket_shift,
  max_mcache_size);
+
+/*
+ * Grant mappings must use XC_PAGE_SIZE granularity since we can't
+ * map anything beyond the number of pages granted to us.
+ */
+mapcache_grants = xen_map_cache_init_single(f, opaque,
+XC_PAGE_SHIFT,
+max_mcache_size);
+
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -168,17 +189,24 @@ static void xen_remap_bucket(MapCache *mc,
  hwaddr size,
  hwaddr address_index,
  bool dummy,
+ bool grant,
+ bool is_write,
  ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
-xen_pfn_t *pfns;
+uint32_t *refs = NULL;
+xen_pfn_t *pfns = NULL;
 int *err;
 unsigned int i;
 hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
 trace_xen_remap_bucket(address_index);
 
-pfns = g_new0(xen_pfn_t, nb_pfn);
+if (grant) {
+refs = g_new0(uint32_t, nb_pfn);
+} else {
+pfns = g_new0(xen_pfn_t, nb_pfn);
+}
 err = g_new0(int, nb_pfn);
 
 if (entry->vaddr_base != NULL) {
@@ -207,21 +235,51 @@ static void xen_remap_bucket(MapCache *mc,
 g_free(entry->valid_mapping);
 entry->valid_mapping = NULL;
 
-for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
+if (grant) {
+hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT);
+
+for (i = 0; i < nb_pfn; i++) {
+refs[i] = grant_base + i;
+}
+} else {
+for (i = 0; i < nb_pfn; i++) {
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_S

[PATCH v6 3/8] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add xen_mr_is_memory() to abstract away tests for the
xen_memory MR.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
Acked-by: David Hildenbrand 
---
 hw/xen/xen-hvm-common.c | 10 --
 include/sysemu/xen.h|  8 
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index 2d1b032121..a0a0252da0 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -12,6 +12,12 @@
 
 MemoryRegion xen_memory;
 
+/* Check for xen memory.  */
+bool xen_mr_is_memory(MemoryRegion *mr)
+{
+return mr == &xen_memory;
+}
+
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
Error **errp)
 {
@@ -28,7 +34,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, 
MemoryRegion *mr,
 return;
 }
 
-if (mr == &xen_memory) {
+if (xen_mr_is_memory(mr)) {
 return;
 }
 
@@ -55,7 +61,7 @@ static void xen_set_memory(struct MemoryListener *listener,
 {
 XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-if (section->mr == &xen_memory) {
+if (xen_mr_is_memory(section->mr)) {
 return;
 } else {
 if (add) {
diff --git a/include/sysemu/xen.h b/include/sysemu/xen.h
index 754ec2e6cb..dc72f83bcb 100644
--- a/include/sysemu/xen.h
+++ b/include/sysemu/xen.h
@@ -34,6 +34,8 @@ void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t 
length);
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
struct MemoryRegion *mr, Error **errp);
 
+bool xen_mr_is_memory(MemoryRegion *mr);
+
 #else /* !CONFIG_XEN_IS_POSSIBLE */
 
 #define xen_enabled() 0
@@ -47,6 +49,12 @@ static inline void xen_ram_alloc(ram_addr_t ram_addr, 
ram_addr_t size,
 g_assert_not_reached();
 }
 
+static inline bool xen_mr_is_memory(MemoryRegion *mr)
+{
+g_assert_not_reached();
+return false;
+}
+
 #endif /* CONFIG_XEN_IS_POSSIBLE */
 
 #endif
-- 
2.40.1

[PATCH v6 1/8] xen: mapcache: Make MCACHE_BUCKET_SHIFT runtime configurable

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Make MCACHE_BUCKET_SHIFT runtime configurable per cache instance.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 54 ++-
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index fa6813b1ad..bc860f4373 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -23,13 +23,10 @@
 
 
 #if HOST_LONG_BITS == 32
-#  define MCACHE_BUCKET_SHIFT 16
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
 #else
-#  define MCACHE_BUCKET_SHIFT 20
 #  define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
 #endif
-#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
 
 /* This is the size of the virtual address space reserve to QEMU that will not
  * be use by MapCache.
@@ -65,7 +62,8 @@ typedef struct MapCache {
 /* For most cases (>99.9%), the page address is the same. */
 MapCacheEntry *last_entry;
 unsigned long max_mcache_size;
-unsigned int mcache_bucket_shift;
+unsigned int bucket_shift;
+unsigned long bucket_size;
 
 phys_offset_to_gaddr_t phys_offset_to_gaddr;
 QemuMutex lock;
@@ -95,11 +93,14 @@ static inline int test_bits(int nr, int size, const 
unsigned long *addr)
 
 static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f,
void *opaque,
+   unsigned int bucket_shift,
unsigned long max_size)
 {
 unsigned long size;
 MapCache *mc;
 
+assert(bucket_shift >= XC_PAGE_SHIFT);
+
 mc = g_new0(MapCache, 1);
 
 mc->phys_offset_to_gaddr = f;
@@ -108,12 +109,14 @@ static MapCache 
*xen_map_cache_init_single(phys_offset_to_gaddr_t f,
 
 QTAILQ_INIT(&mc->locked_entries);
 
+mc->bucket_shift = bucket_shift;
+mc->bucket_size = 1UL << bucket_shift;
 mc->max_mcache_size = max_size;
 
 mc->nr_buckets =
 (((mc->max_mcache_size >> XC_PAGE_SHIFT) +
-  (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
- (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
+  (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >>
+ (bucket_shift - XC_PAGE_SHIFT));
 
 size = mc->nr_buckets * sizeof(MapCacheEntry);
 size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
@@ -126,6 +129,13 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 {
 struct rlimit rlimit_as;
 unsigned long max_mcache_size;
+unsigned int bucket_shift;
+
+if (HOST_LONG_BITS == 32) {
+bucket_shift = 16;
+} else {
+bucket_shift = 20;
+}
 
 if (geteuid() == 0) {
 rlimit_as.rlim_cur = RLIM_INFINITY;
@@ -146,7 +156,9 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 }
 }
 
-mapcache = xen_map_cache_init_single(f, opaque, max_mcache_size);
+mapcache = xen_map_cache_init_single(f, opaque,
+ bucket_shift,
+ max_mcache_size);
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -195,7 +207,7 @@ static void xen_remap_bucket(MapCache *mc,
 entry->valid_mapping = NULL;
 
 for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
 }
 
 /*
@@ -266,8 +278,8 @@ static uint8_t *xen_map_cache_unlocked(MapCache *mc,
 bool dummy = false;
 
 tryagain:
-address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
-address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
+address_index  = phys_addr >> mc->bucket_shift;
+address_offset = phys_addr & (mc->bucket_size - 1);
 
 trace_xen_map_cache(phys_addr);
 
@@ -294,14 +306,14 @@ tryagain:
 return mc->last_entry->vaddr_base + address_offset;
 }
 
-/* size is always a multiple of MCACHE_BUCKET_SIZE */
+/* size is always a multiple of mc->bucket_size */
 if (size) {
 cache_size = size + address_offset;
-if (cache_size % MCACHE_BUCKET_SIZE) {
-cache_size += MCACHE_BUCKET_SIZE - (cache_size % 
MCACHE_BUCKET_SIZE);
+if (cache_size % mc->bucket_size) {
+cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
 }
 } else {
-cache_size = MCACHE_BUCKET_SIZE;
+cache_size = mc->bucket_size;
 }
 
 entry = &mc->entry[address_index % mc->nr_buckets];
@@ -422,7 +434,7 @@ static ram_addr_t 
xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr)
 trace_xen_ram_addr_from_mapcache_not_in_cache(ptr);
 raddr = RAM_ADDR_INVALID;
 } else {
-raddr = (reventry->paddr_index << MCACHE_B

Re: [PATCH v5 6/8] xen: mapcache: Pass the ram_addr offset to xen_map_cache()

[Edgar E. Iglesias]

On Thu, May 16, 2024 at 1:08 AM Stefano Stabellini
 wrote:
>
> On Fri, 3 May 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > Pass the ram_addr offset to xen_map_cache.
> > This is in preparation for adding grant mappings that need
> > to compute the address within the RAMBlock.
> >
> > No functional changes.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/xen/xen-mapcache.c | 16 +++-
> >  include/sysemu/xen-mapcache.h |  2 ++
> >  system/physmem.c  |  9 +
> >  3 files changed, 18 insertions(+), 9 deletions(-)
> >
> > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > index ec95445696..26bc38a9e3 100644
> > --- a/hw/xen/xen-mapcache.c
> > +++ b/hw/xen/xen-mapcache.c
> > @@ -167,7 +167,8 @@ static void xen_remap_bucket(MapCache *mc,
> >   void *vaddr,
> >   hwaddr size,
> >   hwaddr address_index,
> > - bool dummy)
> > + bool dummy,
> > + ram_addr_t ram_offset)
> >  {
> >  uint8_t *vaddr_base;
> >  xen_pfn_t *pfns;
> > @@ -266,6 +267,7 @@ static void xen_remap_bucket(MapCache *mc,
> >
> >  static uint8_t *xen_map_cache_unlocked(MapCache *mc,
> > hwaddr phys_addr, hwaddr size,
> > +   ram_addr_t ram_offset,
> > uint8_t lock, bool dma, bool 
> > is_write)
> >  {
> >  MapCacheEntry *entry, *pentry = NULL,
> > @@ -337,14 +339,16 @@ tryagain:
> >  if (!entry) {
> >  entry = g_new0(MapCacheEntry, 1);
> >  pentry->next = entry;
> > -xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
> > dummy);
> > +xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
> > + ram_offset);
> >  } else if (!entry->lock) {
> >  if (!entry->vaddr_base || entry->paddr_index != address_index ||
> >  entry->size != cache_size ||
> >  !test_bits(address_offset >> XC_PAGE_SHIFT,
> >  test_bit_size >> XC_PAGE_SHIFT,
> >  entry->valid_mapping)) {
> > -xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
> > dummy);
> > +xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
> > dummy,
> > + ram_offset);
> >  }
> >  }
> >
> > @@ -391,13 +395,15 @@ tryagain:
> >
> >  uint8_t *xen_map_cache(MemoryRegion *mr,
> > hwaddr phys_addr, hwaddr size,
> > +   ram_addr_t ram_addr_offset,
> > uint8_t lock, bool dma,
> > bool is_write)
> >  {
> >  uint8_t *p;
> >
> >  mapcache_lock(mapcache);
> > -p = xen_map_cache_unlocked(mapcache, phys_addr, size, lock, dma, 
> > is_write);
> > +p = xen_map_cache_unlocked(mapcache, phys_addr, size, ram_addr_offset,
> > +   lock, dma, is_write);
> >  mapcache_unlock(mapcache);
> >  return p;
> >  }
> > @@ -632,7 +638,7 @@ static uint8_t 
> > *xen_replace_cache_entry_unlocked(MapCache *mc,
> >  trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
> >
> >  xen_remap_bucket(mc, entry, entry->vaddr_base,
> > - cache_size, address_index, false);
> > + cache_size, address_index, false, new_phys_addr);
>
> Everything else makes sense, but I don't understand how can it be that
> new_phys_addr is the block->offset here?
>

Agreed, this should be old_phys_addr propagated via phys_offset from
xen_add_to_physmap().
The reason this didn't have an effect here is that we currently only
use the ram_addr offset for grants,
and xen_replace_cache_entry is never called for grants. Anyway, I'll
fix this in the next version!




>
> >  if (!test_bits(address_offset >> XC_PAGE_SHIFT,
> >  test_bit_size >> XC_PAGE_SHIFT,
> >  entry->valid_mapping)) {
> > diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
> > index 1ec9e66752..b5e3ea1bc0 100644
> > --- a/include/sysemu/xen-mapcache.h
> > +++ b/i

[PATCH v4 6/9] xen/arm64: bpi: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
---
 xen/arch/arm/arm64/bpi.S | 20 
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/xen/arch/arm/arm64/bpi.S b/xen/arch/arm/arm64/bpi.S
index 4e63825220..d6b8defb80 100644
--- a/xen/arch/arm/arm64/bpi.S
+++ b/xen/arch/arm/arm64/bpi.S
@@ -52,14 +52,15 @@
  * micro-architectures in a system.
  */
 .align 11
-ENTRY(__bp_harden_hyp_vecs_start)
+FUNC(__bp_harden_hyp_vecs_start)
 .rept 4
 vectors hyp_traps_vector
 .endr
-ENTRY(__bp_harden_hyp_vecs_end)
+LABEL(__bp_harden_hyp_vecs_end)
+END(__bp_harden_hyp_vecs_start)
 
 .macro mitigate_spectre_bhb_loop count
-ENTRY(__mitigate_spectre_bhb_loop_start_\count)
+FUNC(__mitigate_spectre_bhb_loop_start_\count)
 stp x0, x1, [sp, #-16]!
 mov x0, \count
 .Lspectre_bhb_loop\@:
@@ -68,11 +69,12 @@ ENTRY(__mitigate_spectre_bhb_loop_start_\count)
 b.ne.Lspectre_bhb_loop\@
 sb
 ldp x0, x1, [sp], #16
-ENTRY(__mitigate_spectre_bhb_loop_end_\count)
+LABEL(__mitigate_spectre_bhb_loop_end_\count)
+END(__mitigate_spectre_bhb_loop_start_\count)
 .endm
 
 .macro smccc_workaround num smcc_id
-ENTRY(__smccc_workaround_smc_start_\num)
+FUNC(__smccc_workaround_smc_start_\num)
 sub sp, sp, #(8 * 4)
 stp x0, x1, [sp, #(8 * 2)]
 stp x2, x3, [sp, #(8 * 0)]
@@ -81,13 +83,15 @@ ENTRY(__smccc_workaround_smc_start_\num)
 ldp x2, x3, [sp, #(8 * 0)]
 ldp x0, x1, [sp, #(8 * 2)]
 add sp, sp, #(8 * 4)
-ENTRY(__smccc_workaround_smc_end_\num)
+LABEL(__smccc_workaround_smc_end_\num)
+END(__smccc_workaround_smc_start_\num)
 .endm
 
-ENTRY(__mitigate_spectre_bhb_clear_insn_start)
+FUNC(__mitigate_spectre_bhb_clear_insn_start)
 clearbhb
 isb
-ENTRY(__mitigate_spectre_bhb_clear_insn_end)
+LABEL(__mitigate_spectre_bhb_clear_insn_end)
+END(__mitigate_spectre_bhb_clear_insn_start)
 
 mitigate_spectre_bhb_loop 8
 mitigate_spectre_bhb_loop 24
-- 
2.40.1

[PATCH v4 2/9] xen/arm64: smc: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 xen/arch/arm/arm64/smc.S | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/arm64/smc.S b/xen/arch/arm/arm64/smc.S
index fc6b676e2e..68b05e8ddd 100644
--- a/xen/arch/arm/arm64/smc.S
+++ b/xen/arch/arm/arm64/smc.S
@@ -19,7 +19,7 @@
  *  register_t a6, register_t a7,
  *  struct arm_smccc_res *res)
  */
-ENTRY(__arm_smccc_1_0_smc)
+FUNC(__arm_smccc_1_0_smc)
 smc #0
 ldr x4, [sp]
 cbz x4, 1f  /* No need to store the result */
@@ -27,12 +27,13 @@ ENTRY(__arm_smccc_1_0_smc)
 stp x2, x3, [x4, #SMCCC_RES_a2]
 1:
 ret
+END(__arm_smccc_1_0_smc)
 
 /*
  * void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args,
  *struct arm_smccc_1_2_regs *res)
  */
-ENTRY(arm_smccc_1_2_smc)
+FUNC(arm_smccc_1_2_smc)
 /* Save `res` and free a GPR that won't be clobbered by SMC call */
 stp x1, x19, [sp, #-16]!
 
@@ -69,3 +70,4 @@ ENTRY(arm_smccc_1_2_smc)
 /* Restore original x19 */
 ldp xzr, x19, [sp], #16
 ret
+END(arm_smccc_1_2_smc)
-- 
2.40.1

[PATCH v4 8/9] xen/arm64: cache: Use the generic xen/linkage.h macros

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 xen/arch/arm/arm64/cache.S | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/arm64/cache.S b/xen/arch/arm/arm64/cache.S
index 9a88a2b497..66ed85f735 100644
--- a/xen/arch/arm/arm64/cache.S
+++ b/xen/arch/arm/arm64/cache.S
@@ -40,7 +40,7 @@
  * - kaddr   - kernel address
  * - size- size in question
  */
-ENTRY(__flush_dcache_area)
+FUNC(__flush_dcache_area)
dcache_line_size x2, x3
add x1, x0, x1
sub x3, x2, #1
@@ -51,4 +51,4 @@ ENTRY(__flush_dcache_area)
b.lo1b
dsb sy
ret
-ENDPROC(__flush_dcache_area)
+END(__flush_dcache_area)
-- 
2.40.1

[PATCH v4 1/9] xen/arm64: entry: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
---
 xen/arch/arm/arm64/entry.S | 78 --
 1 file changed, 50 insertions(+), 28 deletions(-)

diff --git a/xen/arch/arm/arm64/entry.S b/xen/arch/arm/arm64/entry.S
index f963c923bb..6251135ebd 100644
--- a/xen/arch/arm/arm64/entry.S
+++ b/xen/arch/arm/arm64/entry.S
@@ -289,21 +289,25 @@
 b   do_bad_mode
 .endm
 
-hyp_sync_invalid:
+FUNC_LOCAL(hyp_sync_invalid)
 entry   hyp=1
 invalid BAD_SYNC
+END(hyp_sync_invalid)
 
-hyp_irq_invalid:
+FUNC_LOCAL(hyp_irq_invalid)
 entry   hyp=1
 invalid BAD_IRQ
+END(hyp_irq_invalid)
 
-hyp_fiq_invalid:
+FUNC_LOCAL(hyp_fiq_invalid)
 entry   hyp=1
 invalid BAD_FIQ
+END(hyp_fiq_invalid)
 
-hyp_error_invalid:
+FUNC_LOCAL(hyp_error_invalid)
 entry   hyp=1
 invalid BAD_ERROR
+END(hyp_error_invalid)
 
 /*
  * SError received while running in the hypervisor mode.
@@ -313,11 +317,12 @@ hyp_error_invalid:
  * simplicity, as SError should be rare and potentially fatal,
  * all interrupts are kept masked.
  */
-hyp_error:
+FUNC_LOCAL(hyp_error)
 entry   hyp=1
 mov x0, sp
 bl  do_trap_hyp_serror
 exithyp=1
+END(hyp_error)
 
 /*
  * Synchronous exception received while running in the hypervisor mode.
@@ -327,7 +332,7 @@ hyp_error:
  * some of them. So we want to inherit the state from the interrupted
  * context.
  */
-hyp_sync:
+FUNC_LOCAL(hyp_sync)
 entry   hyp=1
 
 /* Inherit interrupts */
@@ -338,6 +343,7 @@ hyp_sync:
 mov x0, sp
 bl  do_trap_hyp_sync
 exithyp=1
+END(hyp_sync)
 
 /*
  * IRQ received while running in the hypervisor mode.
@@ -352,7 +358,7 @@ hyp_sync:
  * would require some rework in some paths (e.g. panic, livepatch) to
  * ensure the ordering is enforced everywhere.
  */
-hyp_irq:
+FUNC_LOCAL(hyp_irq)
 entry   hyp=1
 
 /* Inherit D, A, F interrupts and keep I masked */
@@ -365,8 +371,9 @@ hyp_irq:
 mov x0, sp
 bl  do_trap_irq
 exithyp=1
+END(hyp_irq)
 
-guest_sync:
+FUNC_LOCAL(guest_sync)
 /*
  * Save x0, x1 in advance
  */
@@ -413,8 +420,9 @@ fastpath_out_workaround:
 mov x1, xzr
 eret
 sb
+END(guest_sync)
 
-wa2_ssbd:
+FUNC_LOCAL(wa2_ssbd)
 #ifdef CONFIG_ARM_SSBD
 alternative_cb arm_enable_wa2_handling
 b   wa2_end
@@ -450,42 +458,55 @@ wa2_end:
 mov x0, xzr
 eret
 sb
-guest_sync_slowpath:
+END(wa2_ssbd)
+
+FUNC_LOCAL(guest_sync_slowpath)
 /*
  * x0/x1 may have been scratch by the fast path above, so avoid
  * to save them.
  */
 guest_vector compat=0, iflags=IFLAGS__AI_, trap=guest_sync, 
save_x0_x1=0
+END(guest_sync_slowpath)
 
-guest_irq:
+FUNC_LOCAL(guest_irq)
 guest_vector compat=0, iflags=IFLAGS__A__, trap=irq
+END(guest_irq)
 
-guest_fiq_invalid:
+FUNC_LOCAL(guest_fiq_invalid)
 entry   hyp=0, compat=0
 invalid BAD_FIQ
+END(guest_fiq_invalid)
 
-guest_error:
+FUNC_LOCAL(guest_error)
 guest_vector compat=0, iflags=IFLAGS__AI_, trap=guest_serror
+END(guest_error)
 
-guest_sync_compat:
+FUNC_LOCAL(guest_sync_compat)
 guest_vector compat=1, iflags=IFLAGS__AI_, trap=guest_sync
+END(guest_sync_compat)
 
-guest_irq_compat:
+FUNC_LOCAL(guest_irq_compat)
 guest_vector compat=1, iflags=IFLAGS__A__, trap=irq
+END(guest_irq_compat)
 
-guest_fiq_invalid_compat:
+FUNC_LOCAL(guest_fiq_invalid_compat)
 entry   hyp=0, compat=1
 invalid BAD_FIQ
+END(guest_fiq_invalid_compat)
 
-guest_error_compat:
+FUNC_LOCAL(guest_error_compat)
 guest_vector compat=1, iflags=IFLAGS__AI_, trap=guest_serror
+END(guest_error_compat)
 
-ENTRY(return_to_new_vcpu32)
+FUNC(return_to_new_vcpu32)
 exithyp=0, compat=1
-ENTRY(return_to_new_vcpu64)
+END(return_to_new_vcpu32)
+
+FUNC(return_to_new_vcpu64)
 exithyp=0, compat=0
+END(return_to_new_vcpu64)
 
-return_from_trap:
+FUNC_LOCAL(return_from_trap)
 msr daifset, #IFLAGS___I_ /* Mask interrupts */
 
 ldr x21, [sp, #UREGS_PC]/* load ELR */
@@ -524,6 +545,7 @@ return_from_trap:
 
 eret
 sb
+END(return_from_trap)
 
 /*
  * Consume pending SError generated by the guest if any.
@@ -536,7 +558,7 @@ return_from_trap:
  * it. So the function will unmask SError exception for a small window and
  * then mask it again.
  */
-check_pending_guest_serror:
+FUNC_LOCAL(check_pending_guest_serror)
 /*
  * Save elr_el2 to check whether the pending SError exception takes
  * place while we are doing this sync exception.
@@ -562,13 +584,11 @@ check_pending_guest_serror:
  * exception handler, and the elr_el2 will

[PATCH v4 4/9] xen/arm64: head: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
---
 xen/arch/arm/arm64/head.S | 51 +--
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/xen/arch/arm/arm64/head.S b/xen/arch/arm/arm64/head.S
index fb297e9eb5..2fa07dc3a0 100644
--- a/xen/arch/arm/arm64/head.S
+++ b/xen/arch/arm/arm64/head.S
@@ -90,7 +90,7 @@
  * 4K-aligned address.
  */
 
-GLOBAL(start)
+FUNC(start)
 /*
  * DO NOT MODIFY. Image header expected by Linux boot-loaders.
  */
@@ -102,6 +102,7 @@ efi_head:
  */
 add x13, x18, #0x16
 b   real_start   /* branch to kernel start */
+END(start)
 .quad   0/* Image load offset from start of RAM */
 .quad   _end - start /* Effective size of kernel image, 
little-endian */
 .quad   __HEAD_FLAGS /* Informative flags, little-endian */
@@ -223,7 +224,7 @@ section_table:
 .align  5
 #endif /* CONFIG_ARM_EFI */
 
-real_start:
+FUNC_LOCAL(real_start)
 /* BSS should be zeroed when booting without EFI */
 mov   x26, #0/* x26 := skip_zero_bss */
 
@@ -263,9 +264,9 @@ primary_switched:
 mov   x1, x21/* x1 := paddr(FDT) */
 ldr   x2, =start_xen
 b launch
-ENDPROC(real_start)
+END(real_start)
 
-GLOBAL(init_secondary)
+FUNC(init_secondary)
 msr   DAIFSet, 0xf   /* Disable all interrupts */
 
 /* Find out where we are */
@@ -304,7 +305,7 @@ secondary_switched:
 /* Jump to C world */
 ldr   x2, =start_secondary
 b launch
-ENDPROC(init_secondary)
+END(init_secondary)
 
 /*
  * Check if the CPU has been booted in Hypervisor mode.
@@ -313,7 +314,7 @@ ENDPROC(init_secondary)
  *
  * Clobbers x0 - x5
  */
-check_cpu_mode:
+FUNC_LOCAL(check_cpu_mode)
 PRINT_ID("- Current EL ")
 mrs   x5, CurrentEL
 print_reg x5
@@ -329,7 +330,7 @@ check_cpu_mode:
 PRINT_ID("- Xen must be entered in NS EL2 mode -\r\n")
 PRINT_ID("- Please update the bootloader -\r\n")
 b fail
-ENDPROC(check_cpu_mode)
+END(check_cpu_mode)
 
 /*
  * Zero BSS
@@ -339,7 +340,7 @@ ENDPROC(check_cpu_mode)
  *
  * Clobbers x0 - x3
  */
-zero_bss:
+FUNC_LOCAL(zero_bss)
 /* Zero BSS only when requested */
 cbnz  x26, skip_bss
 
@@ -353,14 +354,14 @@ zero_bss:
 
 skip_bss:
 ret
-ENDPROC(zero_bss)
+END(zero_bss)
 
 /*
  * Initialize the processor for turning the MMU on.
  *
  * Clobbers x0 - x3
  */
-cpu_init:
+FUNC_LOCAL(cpu_init)
 PRINT_ID("- Initialize CPU -\r\n")
 
 /* Set up memory attribute type tables */
@@ -399,7 +400,7 @@ cpu_init:
  */
 msr spsel, #1
 ret
-ENDPROC(cpu_init)
+END(cpu_init)
 
 /*
  * Setup the initial stack and jump to the C world
@@ -411,7 +412,7 @@ ENDPROC(cpu_init)
  *
  * Clobbers x3
  */
-launch:
+FUNC_LOCAL(launch)
 ldr   x3, =init_data
 add   x3, x3, #INITINFO_stack /* Find the boot-time stack */
 ldr   x3, [x3]
@@ -421,13 +422,14 @@ launch:
 
 /* Jump to C world */
 brx2
-ENDPROC(launch)
+END(launch)
 
 /* Fail-stop */
-fail:   PRINT_ID("- Boot failed -\r\n")
+FUNC_LOCAL(fail)
+PRINT_ID("- Boot failed -\r\n")
 1:  wfe
 b 1b
-ENDPROC(fail)
+END(fail)
 
 #ifdef CONFIG_EARLY_PRINTK
 /*
@@ -438,14 +440,14 @@ ENDPROC(fail)
  *
  * Clobbers x0 - x1
  */
-init_uart:
+FUNC_LOCAL(init_uart)
 ldr   x23, =CONFIG_EARLY_UART_BASE_ADDRESS
 #ifdef CONFIG_EARLY_UART_INIT
 early_uart_init x23, 0
 #endif
 PRINT("- UART enabled -\r\n")
 ret
-ENDPROC(init_uart)
+END(init_uart)
 
 /*
  * Print early debug messages.
@@ -454,7 +456,7 @@ ENDPROC(init_uart)
  * x23: Early UART base address
  * Clobbers x0-x1
  */
-ENTRY(asm_puts)
+FUNC(asm_puts)
 early_uart_ready x23, 1
 ldrb  w1, [x0], #1   /* Load next char */
 cbz   w1, 1f /* Exit on nul */
@@ -462,7 +464,7 @@ ENTRY(asm_puts)
 b asm_puts
 1:
 ret
-ENDPROC(asm_puts)
+END(asm_puts)
 
 /*
  * Print a 64-bit number in hex.
@@ -471,7 +473,7 @@ ENDPROC(asm_puts)
  * x23: Early UART base address
  * Clobbers x0-x3
  */
-ENTRY(asm_putn)
+FUNC(asm_putn)
 adr_l x1, hex
 mov   x3, #16
 1:
@@ -484,7 +486,7 @@ ENTRY(asm_putn)
 subs  x3, x3, #1
 b.ne  1b
 ret
-ENDPROC(asm_putn)
+END(asm_putn)
 
 RODATA_SECT(.rodata.idmap, hex, "0123456789abcdef")
 
@@ -493,16 +495,17 @@ RODATA_SECT(.rodata.idmap, hex, "0123456789abcdef")
 /* This provides a C-API version of __lookup_processor_type
  * TODO: For now, the implementation return NULL every time
  */
-ENTRY(lookup_processor_type)
+FUNC(lookup_pr

[PATCH v4 9/9] xen/arm64: lib: Use the generic xen/linkage.h macros

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 xen/arch/arm/arm64/lib/memchr.S  | 4 ++--
 xen/arch/arm/arm64/lib/memcmp.S  | 4 ++--
 xen/arch/arm/arm64/lib/memcpy.S  | 4 ++--
 xen/arch/arm/arm64/lib/memmove.S | 4 ++--
 xen/arch/arm/arm64/lib/memset.S  | 4 ++--
 xen/arch/arm/arm64/lib/strchr.S  | 4 ++--
 xen/arch/arm/arm64/lib/strcmp.S  | 4 ++--
 xen/arch/arm/arm64/lib/strlen.S  | 4 ++--
 xen/arch/arm/arm64/lib/strncmp.S | 4 ++--
 xen/arch/arm/arm64/lib/strnlen.S | 4 ++--
 xen/arch/arm/arm64/lib/strrchr.S | 4 ++--
 11 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/xen/arch/arm/arm64/lib/memchr.S b/xen/arch/arm/arm64/lib/memchr.S
index 81f113bb1c..3d8aeca3ca 100644
--- a/xen/arch/arm/arm64/lib/memchr.S
+++ b/xen/arch/arm/arm64/lib/memchr.S
@@ -29,7 +29,7 @@
  * Returns:
  * x0 - address of first occurrence of 'c' or 0
  */
-ENTRY(memchr)
+FUNC(memchr)
and w1, w1, #0xff
 1: subsx2, x2, #1
b.mi2f
@@ -40,4 +40,4 @@ ENTRY(memchr)
ret
 2: mov x0, #0
ret
-ENDPROC(memchr)
+END(memchr)
diff --git a/xen/arch/arm/arm64/lib/memcmp.S b/xen/arch/arm/arm64/lib/memcmp.S
index 87c2537ffe..d77dd4ce52 100644
--- a/xen/arch/arm/arm64/lib/memcmp.S
+++ b/xen/arch/arm/arm64/lib/memcmp.S
@@ -57,7 +57,7 @@ pos   .reqx11
 limit_wd   .reqx12
 mask   .reqx13
 
-ENTRY(memcmp)
+FUNC(memcmp)
cbz limit, .Lret0
eor tmp1, src1, src2
tst tmp1, #7
@@ -254,4 +254,4 @@ CPU_LE( rev data2, data2 )
 .Lret0:
mov result, #0
ret
-ENDPROC(memcmp)
+END(memcmp)
diff --git a/xen/arch/arm/arm64/lib/memcpy.S b/xen/arch/arm/arm64/lib/memcpy.S
index d90d20ef3e..1e04b79010 100644
--- a/xen/arch/arm/arm64/lib/memcpy.S
+++ b/xen/arch/arm/arm64/lib/memcpy.S
@@ -55,7 +55,7 @@ C_h   .reqx12
 D_l.reqx13
 D_h.reqx14
 
-ENTRY(memcpy)
+FUNC(memcpy)
mov dst, dstin
cmp count, #16
/*When memory length is less than 16, the accessed are not aligned.*/
@@ -197,4 +197,4 @@ ENTRY(memcpy)
tst count, #0x3f
b.ne.Ltail63
ret
-ENDPROC(memcpy)
+END(memcpy)
diff --git a/xen/arch/arm/arm64/lib/memmove.S b/xen/arch/arm/arm64/lib/memmove.S
index a49de845d0..14438dbe9c 100644
--- a/xen/arch/arm/arm64/lib/memmove.S
+++ b/xen/arch/arm/arm64/lib/memmove.S
@@ -56,7 +56,7 @@ C_h   .reqx12
 D_l.reqx13
 D_h.reqx14
 
-ENTRY(memmove)
+FUNC(memmove)
cmp dstin, src
b.lomemcpy
add tmp1, src, count
@@ -193,4 +193,4 @@ ENTRY(memmove)
tst count, #0x3f
b.ne.Ltail63
ret
-ENDPROC(memmove)
+END(memmove)
diff --git a/xen/arch/arm/arm64/lib/memset.S b/xen/arch/arm/arm64/lib/memset.S
index 5bf751521b..367fa60175 100644
--- a/xen/arch/arm/arm64/lib/memset.S
+++ b/xen/arch/arm/arm64/lib/memset.S
@@ -53,7 +53,7 @@ dst   .reqx8
 tmp3w  .reqw9
 tmp3   .reqx9
 
-ENTRY(memset)
+FUNC(memset)
mov dst, dstin  /* Preserve return value.  */
and A_lw, val, #255
orr A_lw, A_lw, A_lw, lsl #8
@@ -212,4 +212,4 @@ ENTRY(memset)
andscount, count, zva_bits_x
b.ne.Ltail_maybe_long
ret
-ENDPROC(memset)
+END(memset)
diff --git a/xen/arch/arm/arm64/lib/strchr.S b/xen/arch/arm/arm64/lib/strchr.S
index 0506b0ff7f..83fd81e8ef 100644
--- a/xen/arch/arm/arm64/lib/strchr.S
+++ b/xen/arch/arm/arm64/lib/strchr.S
@@ -27,7 +27,7 @@
  * Returns:
  * x0 - address of first occurrence of 'c' or 0
  */
-ENTRY(strchr)
+FUNC(strchr)
and w1, w1, #0xff
 1: ldrbw2, [x0], #1
cmp w2, w1
@@ -37,4 +37,4 @@ ENTRY(strchr)
cmp w2, w1
cselx0, x0, xzr, eq
ret
-ENDPROC(strchr)
+END(strchr)
diff --git a/xen/arch/arm/arm64/lib/strcmp.S b/xen/arch/arm/arm64/lib/strcmp.S
index c6f42dd255..7677108e26 100644
--- a/xen/arch/arm/arm64/lib/strcmp.S
+++ b/xen/arch/arm/arm64/lib/strcmp.S
@@ -59,7 +59,7 @@ tmp3  .reqx9
 zeroones   .reqx10
 pos.reqx11
 
-ENTRY(strcmp)
+FUNC(strcmp)
eor tmp1, src1, src2
mov zeroones, #REP8_01
tst tmp1, #7
@@ -230,4 +230,4 @@ CPU_BE( orr syndrome, diff, has_nul )
lsr data1, data1, #56
sub result, data1, data2, lsr #56
ret
-ENDPROC(strcmp)
+END(strcmp)
diff --git a/xen/arch/arm/arm64/lib/strlen.S b/xen/arch/arm/arm64/lib/strlen.S
index fb6aaf1a6a..10feedaf81 100644
--- a/xen/arch/arm/arm64/lib/strlen.S
+++ b/xen/arch/arm/arm64/lib/strlen.S
@@ -56,7 +56,7 @@ pos   .reqx12
 #define REP8_7f 0x7f7f7f7f7f7f7f7f
 #define REP8_80 0x8080808080808080
 
-ENTRY(strlen)
+FUNC(strlen)
mov zeroones, #REP8_01
bic src, srcin, #15
andstmp1, srcin, #15
@

[PATCH v4 7/9] xen/arm64: mmu/head: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
---
 xen/arch/arm/arm64/mmu/head.S | 29 +++--
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/xen/arch/arm/arm64/mmu/head.S b/xen/arch/arm/arm64/mmu/head.S
index fa40b696dd..665a51a337 100644
--- a/xen/arch/arm/arm64/mmu/head.S
+++ b/xen/arch/arm/arm64/mmu/head.S
@@ -138,7 +138,7 @@
  *
  * Clobbers x0 - x4
  */
-create_page_tables:
+FUNC_LOCAL(create_page_tables)
 /* Prepare the page-tables for mapping Xen */
 ldr   x0, =XEN_VIRT_START
 create_table_entry boot_pgtable, boot_first, x0, 0, x1, x2, x3
@@ -260,7 +260,7 @@ virtphys_clash:
 /* Identity map clashes with boot_third, which we cannot handle yet */
 PRINT("- Unable to build boot page tables - virt and phys addresses 
clash. -\r\n")
 b fail
-ENDPROC(create_page_tables)
+END(create_page_tables)
 
 /*
  * Turn on the Data Cache and the MMU. The function will return on the 1:1
@@ -273,7 +273,7 @@ ENDPROC(create_page_tables)
  *
  * Clobbers x0 - x5
  */
-enable_mmu:
+FUNC_LOCAL(enable_mmu)
 mov   x4, x0
 mov   x5, x1
 PRINT_ID("- Turning on paging -\r\n")
@@ -304,7 +304,7 @@ enable_mmu:
 PRINT_ID("- Paging turned on -\r\n")
 
 ret
-ENDPROC(enable_mmu)
+END(enable_mmu)
 
 /*
  * Enable mm (turn on the data cache and the MMU) for secondary CPUs.
@@ -316,7 +316,7 @@ ENDPROC(enable_mmu)
  *
  * Clobbers x0 - x6
  */
-ENTRY(enable_secondary_cpu_mm)
+FUNC(enable_secondary_cpu_mm)
 mov   x6, lr
 
 load_paddr x0, init_ttbr
@@ -328,7 +328,7 @@ ENTRY(enable_secondary_cpu_mm)
 
 /* Return to the virtual address requested by the caller. */
 ret
-ENDPROC(enable_secondary_cpu_mm)
+END(enable_secondary_cpu_mm)
 
 /*
  * Enable mm (turn on the data cache and the MMU) for the boot CPU.
@@ -340,7 +340,7 @@ ENDPROC(enable_secondary_cpu_mm)
  *
  * Clobbers x0 - x6
  */
-ENTRY(enable_boot_cpu_mm)
+FUNC(enable_boot_cpu_mm)
 mov   x6, lr
 
 blcreate_page_tables
@@ -365,7 +365,7 @@ ENTRY(enable_boot_cpu_mm)
  * by the caller.
  */
 b remove_identity_mapping
-ENDPROC(enable_boot_cpu_mm)
+END(enable_boot_cpu_mm)
 
 /*
  * Remove the 1:1 map from the page-tables. It is not easy to keep track
@@ -377,7 +377,7 @@ ENDPROC(enable_boot_cpu_mm)
  *
  * Clobbers x0 - x1
  */
-remove_identity_mapping:
+FUNC_LOCAL(remove_identity_mapping)
 /*
  * Find the zeroeth slot used. Remove the entry from zeroeth
  * table if the slot is not XEN_ZEROETH_SLOT.
@@ -419,20 +419,21 @@ identity_mapping_removed:
 flush_xen_tlb_local
 
 ret
-ENDPROC(remove_identity_mapping)
+END(remove_identity_mapping)
 
 /* Fail-stop */
-fail:   PRINT("- Boot failed -\r\n")
+FUNC_LOCAL(fail)
+PRINT("- Boot failed -\r\n")
 1:  wfe
 b 1b
-ENDPROC(fail)
+END(fail)
 
 /*
  * Switch TTBR
  *
  * x0ttbr
  */
-ENTRY(switch_ttbr_id)
+FUNC(switch_ttbr_id)
 /* 1) Ensure any previous read/write have completed */
 dsbish
 isb
@@ -464,7 +465,7 @@ ENTRY(switch_ttbr_id)
 isb
 
 ret
-ENDPROC(switch_ttbr_id)
+END(switch_ttbr_id)
 
 /*
  * Local variables:
-- 
2.40.1

[PATCH v4 3/9] xen/arm64: sve: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 xen/arch/arm/arm64/sve-asm.S | 9 ++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/xen/arch/arm/arm64/sve-asm.S b/xen/arch/arm/arm64/sve-asm.S
index 59dbefbbb2..2d8b895f07 100644
--- a/xen/arch/arm/arm64/sve-asm.S
+++ b/xen/arch/arm/arm64/sve-asm.S
@@ -161,9 +161,10 @@
 .endm
 
 /* Gets the current vector register size in bytes */
-GLOBAL(sve_get_hw_vl)
+FUNC(sve_get_hw_vl)
 _sve_rdvl 0, 1
 ret
+END(sve_get_hw_vl)
 
 /*
  * Save the SVE context
@@ -172,9 +173,10 @@ GLOBAL(sve_get_hw_vl)
  * x1 - pointer to buffer for P0-15
  * x2 - Save FFR if non-zero
  */
-GLOBAL(sve_save_ctx)
+FUNC(sve_save_ctx)
 sve_save 0, 1, x2
 ret
+END(sve_save_ctx)
 
 /*
  * Load the SVE context
@@ -183,9 +185,10 @@ GLOBAL(sve_save_ctx)
  * x1 - pointer to buffer for P0-15
  * x2 - Restore FFR if non-zero
  */
-GLOBAL(sve_load_ctx)
+FUNC(sve_load_ctx)
 sve_load 0, 1, x2
 ret
+END(sve_load_ctx)
 
 /*
  * Local variables:
-- 
2.40.1

[PATCH v4 0/9] xen/arm: arm64: Annotate code symbols

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

On the way towards Xen safety certification we're evaluating the use
of tools to collect code-coverage/profiling information from execution
traces. Some tools rely on ELF symbols for code being declared with
type FUNC and having a symbol size.

We currently annotate some symbols but not all. Also, there seems to be
different ways to do the annotation.

This series now converts all the .S files under xen/arm/arm64 to use
the macros from xen/linkage.h and also adds missing code symbol
annotations for global symbols. There are still some local labels that
are left unmodified.

Best regards,
Edgar

ChangeLog:

v3 -> v4:
* Add newline between FUNC_LOCAL and PRINT_ID.
* Use LABEL instead of GLOBAL in bpi.S.
* Changed .global -> LABEL in entry.S (dropped Stefanos RB tag).

v2 -> v3:
* Convert and add annotations for all of xen/arm/arm64.
* Fix commit message for entry.S.

v1 -> v2:
* Drop RFC.
* Squash into a single patch to avoid confusion.


Edgar E. Iglesias (9):
  xen/arm64: entry: Add missing code symbol annotations
  xen/arm64: smc: Add missing code symbol annotations
  xen/arm64: sve: Add missing code symbol annotations
  xen/arm64: head: Add missing code symbol annotations
  xen/arm64: debug: Add missing code symbol annotations
  xen/arm64: bpi: Add missing code symbol annotations
  xen/arm64: mmu/head: Add missing code symbol annotations
  xen/arm64: cache: Use the generic xen/linkage.h macros
  xen/arm64: lib: Use the generic xen/linkage.h macros

 xen/arch/arm/arm64/bpi.S | 20 
 xen/arch/arm/arm64/cache.S   |  4 +-
 xen/arch/arm/arm64/debug.S   |  6 ++-
 xen/arch/arm/arm64/entry.S   | 78 
 xen/arch/arm/arm64/head.S| 51 +++--
 xen/arch/arm/arm64/lib/memchr.S  |  4 +-
 xen/arch/arm/arm64/lib/memcmp.S  |  4 +-
 xen/arch/arm/arm64/lib/memcpy.S  |  4 +-
 xen/arch/arm/arm64/lib/memmove.S |  4 +-
 xen/arch/arm/arm64/lib/memset.S  |  4 +-
 xen/arch/arm/arm64/lib/strchr.S  |  4 +-
 xen/arch/arm/arm64/lib/strcmp.S  |  4 +-
 xen/arch/arm/arm64/lib/strlen.S  |  4 +-
 xen/arch/arm/arm64/lib/strncmp.S |  4 +-
 xen/arch/arm/arm64/lib/strnlen.S |  4 +-
 xen/arch/arm/arm64/lib/strrchr.S |  4 +-
 xen/arch/arm/arm64/mmu/head.S| 29 ++--
 xen/arch/arm/arm64/smc.S |  6 ++-
 xen/arch/arm/arm64/sve-asm.S |  9 ++--
 19 files changed, 142 insertions(+), 105 deletions(-)

-- 
2.40.1

[PATCH v4 5/9] xen/arm64: debug: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 xen/arch/arm/arm64/debug.S | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/arm64/debug.S b/xen/arch/arm/arm64/debug.S
index 71cad9d762..c3d02c33d7 100644
--- a/xen/arch/arm/arm64/debug.S
+++ b/xen/arch/arm/arm64/debug.S
@@ -27,17 +27,19 @@
  * Print a character on the UART - this function is called by C
  * x0: character to print
  */
-GLOBAL(early_putch)
+FUNC(early_putch)
 ldr   x15, =EARLY_UART_VIRTUAL_ADDRESS
 early_uart_ready x15, 1
 early_uart_transmit x15, w0
 ret
+END(early_putch)
 
 /* Flush the UART - this function is called by C */
-GLOBAL(early_flush)
+FUNC(early_flush)
 ldr   x15, =EARLY_UART_VIRTUAL_ADDRESS  /* x15 := VA UART base address 
*/
 early_uart_ready x15, 1
 ret
+END(early_flush)
 
 /*
  * Local variables:
-- 
2.40.1

Re: [PATCH v3 6/9] xen/arm64: bpi: Add missing code symbol annotations

[Edgar E. Iglesias]

On Tue, May 7, 2024 at 7:37 PM Julien Grall  wrote:
>
>
>
> On 07/05/2024 17:55, Edgar E. Iglesias wrote:
> > On Tue, May 7, 2024 at 11:57 AM Julien Grall  wrote:
> > Hi Julien,
>
> Hi Edgar,
>
> >
> > The reason I choose FUNC for the start of the symbol is because these
> > symbols contain
> > executable code (not only a table of pointers to code somewhere else)
> > and the ELF spec
> > says that STT_FUNC means the symbol contains functions or other executable
> > code (not only callable functions IIUC):
> >
> > "STT_FUNC The symbol is associated with a function or other executable 
> > code."
> > https://refspecs.linuxbase.org/elf/elf.pdf
> > (Symbol Table 1-20).
>
> Thanks for the pointer. I originally did intend to suggest the change,
> but then I saw the use of LABEL in x86 (such as svm_stgi_label). There
> are a few others example with LABEL_LOCAL.
>
> AFAICT, this is also executable code which the only difference that it
> is not meant to be called by someone else. Furthermore, LABEL is using
> DO_CODE_ALIGN(...) for the alignment which imply that it is intended to
> be used by executable code. So I thought the only difference was whether
> the label was intended to be used as a function.
>

Thanks Julien, yes, good points.

> >
> > I think using LABEL instead of GLOBAL for the _end labels of these
> > code sequences makes sense.
> > I'm happy to change the _start labels to LABEL too if you guys feel
> > that's better.
>
> I have to admit I am little confused with the difference between LABEL
> vs FUNC. I think I will need some guidance from Jan (he introduced
> linkage.h).
>

Jan, do you have any guidance on this specific case using FUNC , LABEL
(or something else)?

Cheers,
Edgar

[PATCH v5 6/8] xen: mapcache: Pass the ram_addr offset to xen_map_cache()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Pass the ram_addr offset to xen_map_cache.
This is in preparation for adding grant mappings that need
to compute the address within the RAMBlock.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-mapcache.c | 16 +++-
 include/sysemu/xen-mapcache.h |  2 ++
 system/physmem.c  |  9 +
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index ec95445696..26bc38a9e3 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -167,7 +167,8 @@ static void xen_remap_bucket(MapCache *mc,
  void *vaddr,
  hwaddr size,
  hwaddr address_index,
- bool dummy)
+ bool dummy,
+ ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
 xen_pfn_t *pfns;
@@ -266,6 +267,7 @@ static void xen_remap_bucket(MapCache *mc,
 
 static uint8_t *xen_map_cache_unlocked(MapCache *mc,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_offset,
uint8_t lock, bool dma, bool is_write)
 {
 MapCacheEntry *entry, *pentry = NULL,
@@ -337,14 +339,16 @@ tryagain:
 if (!entry) {
 entry = g_new0(MapCacheEntry, 1);
 pentry->next = entry;
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 } else if (!entry->lock) {
 if (!entry->vaddr_base || entry->paddr_index != address_index ||
 entry->size != cache_size ||
 !test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
-xen_remap_bucket(mc, entry, NULL, cache_size, address_index, 
dummy);
+xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
+ ram_offset);
 }
 }
 
@@ -391,13 +395,15 @@ tryagain:
 
 uint8_t *xen_map_cache(MemoryRegion *mr,
hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write)
 {
 uint8_t *p;
 
 mapcache_lock(mapcache);
-p = xen_map_cache_unlocked(mapcache, phys_addr, size, lock, dma, is_write);
+p = xen_map_cache_unlocked(mapcache, phys_addr, size, ram_addr_offset,
+   lock, dma, is_write);
 mapcache_unlock(mapcache);
 return p;
 }
@@ -632,7 +638,7 @@ static uint8_t *xen_replace_cache_entry_unlocked(MapCache 
*mc,
 trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
 
 xen_remap_bucket(mc, entry, entry->vaddr_base,
- cache_size, address_index, false);
+ cache_size, address_index, false, new_phys_addr);
 if (!test_bits(address_offset >> XC_PAGE_SHIFT,
 test_bit_size >> XC_PAGE_SHIFT,
 entry->valid_mapping)) {
diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index 1ec9e66752..b5e3ea1bc0 100644
--- a/include/sysemu/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -19,6 +19,7 @@ typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr phys_offset,
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
 void *opaque);
 uint8_t *xen_map_cache(MemoryRegion *mr, hwaddr phys_addr, hwaddr size,
+   ram_addr_t ram_addr_offset,
uint8_t lock, bool dma,
bool is_write);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
@@ -37,6 +38,7 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t 
f,
 static inline uint8_t *xen_map_cache(MemoryRegion *mr,
  hwaddr phys_addr,
  hwaddr size,
+ ram_addr_t ram_addr_offset,
  uint8_t lock,
  bool dma,
  bool is_write)
diff --git a/system/physmem.c b/system/physmem.c
index b7847db1a2..33d09f7571 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2231,13 +2231,14 @@ static void *qemu_ram_ptr_length(RAMBlock *block, 
ram_addr_t addr,
  */
 if (xen_mr_is_memory(block->mr)) {
 return xen_map_cache(block->mr, block->offset + addr,
- len, lock, lock,
- is_write);
+ len, block->offset,
+ lock, lock, is_write);

[PATCH v5 2/8] xen: mapcache: Unmap first entries in buckets

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

When invalidating memory ranges, if we happen to hit the first
entry in a bucket we were never unmapping it. This was harmless
for foreign mappings but now that we're looking to reuse the
mapcache for transient grant mappings, we must unmap entries
when invalidated.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index bc860f4373..ec95445696 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -491,18 +491,23 @@ static void 
xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
 return;
 }
 entry->lock--;
-if (entry->lock > 0 || pentry == NULL) {
+if (entry->lock > 0) {
 return;
 }
 
-pentry->next = entry->next;
 ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
 if (munmap(entry->vaddr_base, entry->size) != 0) {
 perror("unmap fails");
 exit(-1);
 }
+
 g_free(entry->valid_mapping);
-g_free(entry);
+if (pentry) {
+pentry->next = entry->next;
+g_free(entry);
+} else {
+memset(entry, 0, sizeof *entry);
+}
 }
 
 typedef struct XenMapCacheData {
-- 
2.40.1

[PATCH v5 7/8] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add a second mapcache for grant mappings. The mapcache for
grants needs to work with XC_PAGE_SIZE granularity since
we can't map larger ranges than what has been granted to us.

Like with foreign mappings (xen_memory), machines using grants
are expected to initialize the xen_grants MR and map it
into their address-map accordingly.

Signed-off-by: Edgar E. Iglesias 
---
 hw/xen/xen-hvm-common.c |  12 ++-
 hw/xen/xen-mapcache.c   | 163 ++--
 include/hw/xen/xen-hvm-common.h |   3 +
 include/sysemu/xen.h|   7 ++
 4 files changed, 152 insertions(+), 33 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index c94f1990c5..7a1e2ce4b3 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -10,12 +10,18 @@
 #include "hw/boards.h"
 #include "hw/xen/arch_hvm.h"
 
-MemoryRegion xen_memory;
+MemoryRegion xen_memory, xen_grants;
 
-/* Check for xen memory.  */
+/* Check for any kind of xen memory, foreign mappings or grants.  */
 bool xen_mr_is_memory(MemoryRegion *mr)
 {
-return mr == &xen_memory;
+return mr == &xen_memory || mr == &xen_grants;
+}
+
+/* Check specifically for grants.  */
+bool xen_mr_is_grants(MemoryRegion *mr)
+{
+return mr == &xen_grants;
 }
 
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index 26bc38a9e3..25041ab02d 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -14,6 +14,7 @@
 
 #include 
 
+#include "hw/xen/xen-hvm-common.h"
 #include "hw/xen/xen_native.h"
 #include "qemu/bitmap.h"
 
@@ -21,6 +22,8 @@
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
 
+#include 
+#include 
 
 #if HOST_LONG_BITS == 32
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
@@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
 unsigned long *valid_mapping;
 uint32_t lock;
 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
+#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
 uint8_t flags;
 hwaddr size;
 struct MapCacheEntry *next;
@@ -71,6 +75,8 @@ typedef struct MapCache {
 } MapCache;
 
 static MapCache *mapcache;
+static MapCache *mapcache_grants;
+static xengnttab_handle *xen_region_gnttabdev;
 
 static inline void mapcache_lock(MapCache *mc)
 {
@@ -131,6 +137,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 unsigned long max_mcache_size;
 unsigned int bucket_shift;
 
+xen_region_gnttabdev = xengnttab_open(NULL, 0);
+if (xen_region_gnttabdev == NULL) {
+error_report("mapcache: Failed to open gnttab device");
+exit(EXIT_FAILURE);
+}
+
 if (HOST_LONG_BITS == 32) {
 bucket_shift = 16;
 } else {
@@ -159,6 +171,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 mapcache = xen_map_cache_init_single(f, opaque,
  bucket_shift,
  max_mcache_size);
+
+/*
+ * Grant mappings must use XC_PAGE_SIZE granularity since we can't
+ * map anything beyond the number of pages granted to us.
+ */
+mapcache_grants = xen_map_cache_init_single(f, opaque,
+XC_PAGE_SHIFT,
+max_mcache_size);
+
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -168,17 +189,24 @@ static void xen_remap_bucket(MapCache *mc,
  hwaddr size,
  hwaddr address_index,
  bool dummy,
+ bool grant,
+ bool is_write,
  ram_addr_t ram_offset)
 {
 uint8_t *vaddr_base;
-xen_pfn_t *pfns;
+uint32_t *refs = NULL;
+xen_pfn_t *pfns = NULL;
 int *err;
 unsigned int i;
 hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
 trace_xen_remap_bucket(address_index);
 
-pfns = g_new0(xen_pfn_t, nb_pfn);
+if (grant) {
+refs = g_new0(uint32_t, nb_pfn);
+} else {
+pfns = g_new0(xen_pfn_t, nb_pfn);
+}
 err = g_new0(int, nb_pfn);
 
 if (entry->vaddr_base != NULL) {
@@ -207,21 +235,51 @@ static void xen_remap_bucket(MapCache *mc,
 g_free(entry->valid_mapping);
 entry->valid_mapping = NULL;
 
-for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
+if (grant) {
+hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT);
+
+for (i = 0; i < nb_pfn; i++) {
+refs[i] = grant_base + i;
+}
+} else {
+for (i = 0; i < nb_pfn; i++) {
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + 
i;
+}
 }
 
-/*

[PATCH v5 3/8] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Add xen_mr_is_memory() to abstract away tests for the
xen_memory MR.

No functional changes.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
Acked-by: David Hildenbrand 
---
 hw/xen/xen-hvm-common.c | 10 --
 include/sysemu/xen.h|  8 
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
index 1627da7398..c94f1990c5 100644
--- a/hw/xen/xen-hvm-common.c
+++ b/hw/xen/xen-hvm-common.c
@@ -12,6 +12,12 @@
 
 MemoryRegion xen_memory;
 
+/* Check for xen memory.  */
+bool xen_mr_is_memory(MemoryRegion *mr)
+{
+return mr == &xen_memory;
+}
+
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
Error **errp)
 {
@@ -28,7 +34,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, 
MemoryRegion *mr,
 return;
 }
 
-if (mr == &xen_memory) {
+if (xen_mr_is_memory(mr)) {
 return;
 }
 
@@ -55,7 +61,7 @@ static void xen_set_memory(struct MemoryListener *listener,
 {
 XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-if (section->mr == &xen_memory) {
+if (xen_mr_is_memory(section->mr)) {
 return;
 } else {
 if (add) {
diff --git a/include/sysemu/xen.h b/include/sysemu/xen.h
index 754ec2e6cb..dc72f83bcb 100644
--- a/include/sysemu/xen.h
+++ b/include/sysemu/xen.h
@@ -34,6 +34,8 @@ void xen_hvm_modified_memory(ram_addr_t start, ram_addr_t 
length);
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
struct MemoryRegion *mr, Error **errp);
 
+bool xen_mr_is_memory(MemoryRegion *mr);
+
 #else /* !CONFIG_XEN_IS_POSSIBLE */
 
 #define xen_enabled() 0
@@ -47,6 +49,12 @@ static inline void xen_ram_alloc(ram_addr_t ram_addr, 
ram_addr_t size,
 g_assert_not_reached();
 }
 
+static inline bool xen_mr_is_memory(MemoryRegion *mr)
+{
+g_assert_not_reached();
+return false;
+}
+
 #endif /* CONFIG_XEN_IS_POSSIBLE */
 
 #endif
-- 
2.40.1

[PATCH v5 1/8] xen: mapcache: Make MCACHE_BUCKET_SHIFT runtime configurable

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Make MCACHE_BUCKET_SHIFT runtime configurable per cache instance.

Signed-off-by: Edgar E. Iglesias 
Reviewed-by: Stefano Stabellini 
---
 hw/xen/xen-mapcache.c | 54 ++-
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
index fa6813b1ad..bc860f4373 100644
--- a/hw/xen/xen-mapcache.c
+++ b/hw/xen/xen-mapcache.c
@@ -23,13 +23,10 @@
 
 
 #if HOST_LONG_BITS == 32
-#  define MCACHE_BUCKET_SHIFT 16
 #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
 #else
-#  define MCACHE_BUCKET_SHIFT 20
 #  define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
 #endif
-#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
 
 /* This is the size of the virtual address space reserve to QEMU that will not
  * be use by MapCache.
@@ -65,7 +62,8 @@ typedef struct MapCache {
 /* For most cases (>99.9%), the page address is the same. */
 MapCacheEntry *last_entry;
 unsigned long max_mcache_size;
-unsigned int mcache_bucket_shift;
+unsigned int bucket_shift;
+unsigned long bucket_size;
 
 phys_offset_to_gaddr_t phys_offset_to_gaddr;
 QemuMutex lock;
@@ -95,11 +93,14 @@ static inline int test_bits(int nr, int size, const 
unsigned long *addr)
 
 static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f,
void *opaque,
+   unsigned int bucket_shift,
unsigned long max_size)
 {
 unsigned long size;
 MapCache *mc;
 
+assert(bucket_shift >= XC_PAGE_SHIFT);
+
 mc = g_new0(MapCache, 1);
 
 mc->phys_offset_to_gaddr = f;
@@ -108,12 +109,14 @@ static MapCache 
*xen_map_cache_init_single(phys_offset_to_gaddr_t f,
 
 QTAILQ_INIT(&mc->locked_entries);
 
+mc->bucket_shift = bucket_shift;
+mc->bucket_size = 1UL << bucket_shift;
 mc->max_mcache_size = max_size;
 
 mc->nr_buckets =
 (((mc->max_mcache_size >> XC_PAGE_SHIFT) +
-  (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
- (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
+  (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >>
+ (bucket_shift - XC_PAGE_SHIFT));
 
 size = mc->nr_buckets * sizeof(MapCacheEntry);
 size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
@@ -126,6 +129,13 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 {
 struct rlimit rlimit_as;
 unsigned long max_mcache_size;
+unsigned int bucket_shift;
+
+if (HOST_LONG_BITS == 32) {
+bucket_shift = 16;
+} else {
+bucket_shift = 20;
+}
 
 if (geteuid() == 0) {
 rlimit_as.rlim_cur = RLIM_INFINITY;
@@ -146,7 +156,9 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
*opaque)
 }
 }
 
-mapcache = xen_map_cache_init_single(f, opaque, max_mcache_size);
+mapcache = xen_map_cache_init_single(f, opaque,
+ bucket_shift,
+ max_mcache_size);
 setrlimit(RLIMIT_AS, &rlimit_as);
 }
 
@@ -195,7 +207,7 @@ static void xen_remap_bucket(MapCache *mc,
 entry->valid_mapping = NULL;
 
 for (i = 0; i < nb_pfn; i++) {
-pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
+pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
 }
 
 /*
@@ -266,8 +278,8 @@ static uint8_t *xen_map_cache_unlocked(MapCache *mc,
 bool dummy = false;
 
 tryagain:
-address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
-address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
+address_index  = phys_addr >> mc->bucket_shift;
+address_offset = phys_addr & (mc->bucket_size - 1);
 
 trace_xen_map_cache(phys_addr);
 
@@ -294,14 +306,14 @@ tryagain:
 return mc->last_entry->vaddr_base + address_offset;
 }
 
-/* size is always a multiple of MCACHE_BUCKET_SIZE */
+/* size is always a multiple of mc->bucket_size */
 if (size) {
 cache_size = size + address_offset;
-if (cache_size % MCACHE_BUCKET_SIZE) {
-cache_size += MCACHE_BUCKET_SIZE - (cache_size % 
MCACHE_BUCKET_SIZE);
+if (cache_size % mc->bucket_size) {
+cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
 }
 } else {
-cache_size = MCACHE_BUCKET_SIZE;
+cache_size = mc->bucket_size;
 }
 
 entry = &mc->entry[address_index % mc->nr_buckets];
@@ -422,7 +434,7 @@ static ram_addr_t 
xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr)
 trace_xen_ram_addr_from_mapcache_not_in_cache(ptr);
 raddr = RAM_ADDR_INVALID;
 } else {
-raddr = (reventry->paddr_index << MCACHE_B

Re: [PATCH v4 15/17] xen: mapcache: Remove assumption of RAMBlock with 0 offset

[Edgar E. Iglesias]

On Thu, May 2, 2024 at 10:02 PM Stefano Stabellini
 wrote:
>
> On Thu, 2 May 2024, Edgar E. Iglesias wrote:
> > On Thu, May 2, 2024 at 8:53 PM Stefano Stabellini
> >  wrote:
> > >
> > > +Xenia
> > >
> > > On Thu, 2 May 2024, Edgar E. Iglesias wrote:
> > > > On Wed, May 1, 2024 at 11:24 PM Stefano Stabellini
> > > >  wrote:
> > > > >
> > > > > On Tue, 30 Apr 2024, Edgar E. Iglesias wrote:
> > > > > > From: "Edgar E. Iglesias" 
> > > > > >
> > > > > > The current mapcache assumes that all memory is mapped
> > > > > > in a single RAM MR (the first one with offset 0). Remove
> > > > > > this assumption and propagate the offset to the mapcache
> > > > > > so it can do reverse mappings (from hostptr -> ram_addr).
> > > > > >
> > > > > > This is in preparation for adding grant mappings.
> > > > > >
> > > > > > Signed-off-by: Edgar E. Iglesias 
> > > > >
> > > > >
> > > > > Looking at xen_remap_bucket, it is only using address_index (without
> > > > > adding ram_offset) to map foreign memory. From xen_remap_bucket, I 
> > > > > would
> > > > > understand that address_index already includes the ram_offset.
> > > > >
> > > > > Meaning that if we want to map foreign mapping at address 0x5000, then
> > > > > address_index would be 0x5000, even if ram_offset is 0x1000.
> > > > >
> > > > > But then looking xen_ram_addr_from_mapcache_single ram_offset is added
> > > > > to paddr_index to calculate the physical address. So in that case we
> > > > > would want address_index to be 0x4000 and ram_offset to be 0x1000. But
> > > > > xen_remap_bucket would have to sum address_index and ram_offset to map
> > > > > foreign memory.
> > > > >
> > > > > So I am a bit confused, did I get it wrong? One more comment below.
> > > > >
> > > >
> > > > Thanks Stefano,
> > > >
> > > > I think the confusion is that this ram_addr_offset is not related to
> > > > guest address-space.
> > > > It's a QEMU internal thing and it shouldn't be included in the address
> > > > used to map foreign memory.
> > > > The mapcache can treat this ram_addr offset like a cookie that we keep
> > > > around to be able to do
> > > > reverse mappings from host pointers into ram_addr space
> > > > (xen_ram_addr_from_mapcache).
> > > >
> > > > The current mapcache implementation works because we've really only
> > > > been using foreign mappings
> > > > on RAMBlocks with offset 0. We're also creating RAM's such that the
> > > > offset into the RAM is also
> > > > the guest physical address, for x86 this is natural since RAM starts
> > > > at zero (for lowmem) but for
> > > > ARM we're creating larger than needed RAM's (GUEST_RAM0_BASE + 
> > > > ram-size) to
> > > > make this assumption true. Anyway, In this series I'm not addressing
> > > > this second assumption.
> > >
> > > Let's see if I understand correctly.
> > >
> > > The ram_addr space is an internal QEMU address space which is different
> > > from the guest physical address space and thus cannot and should not be
> > > used to do foreign mappings (foreign mapping hypercalls take a guest
> > > physical or a real physical address to map). Is that correct?
> > >
> > > If so, then I understand.
> > >
> >
> > Yes, that matches my understanding.
> >
> > >
> > >
> > > > There's a second call in physmem.c to xen_map_cache using the
> > > > block->offset as an address.
> > > > I was considering removing that second call since I can't see how it 
> > > > can work
> > > > (except perhaps in some specific use-case by luck?). Anyway, for now
> > > > I've left it unmodified.
> > >
> > > Yes, that code was written with the assumption that block->offset is an
> > > offset in the guest physical address space and could be used as a guest
> > > physical address. Actually, you might have spotted a real bug.
> > >
> > > The intent was for smaller regions (not the bit RAM region, things 

Re: [PATCH v3 6/9] xen/arm64: bpi: Add missing code symbol annotations

[Edgar E. Iglesias]

On Tue, May 7, 2024 at 11:57 AM Julien Grall  wrote:
>
> Hi,
>
> On 06/05/2024 13:54, Edgar E. Iglesias wrote:
> > On Sat, May 4, 2024 at 2:14 AM Stefano Stabellini
> >  wrote:
> >>
> >> On Wed, 1 May 2024, Edgar E. Iglesias wrote:
> >>> From: "Edgar E. Iglesias" 
> >>>
> >>> Use the generic xen/linkage.h macros to annotate code symbols
> >>> and add missing annotations.
> >>>
> >>> Signed-off-by: Edgar E. Iglesias 
> >>> ---
> >>>   xen/arch/arm/arm64/bpi.S | 20 
> >>>   1 file changed, 12 insertions(+), 8 deletions(-)
> >>>
> >>> diff --git a/xen/arch/arm/arm64/bpi.S b/xen/arch/arm/arm64/bpi.S
> >>> index 4e63825220..b16e4d1e29 100644
> >>> --- a/xen/arch/arm/arm64/bpi.S
> >>> +++ b/xen/arch/arm/arm64/bpi.S
> >>> @@ -52,14 +52,15 @@
> >>>* micro-architectures in a system.
> >>>*/
> >>>   .align   11
> >>> -ENTRY(__bp_harden_hyp_vecs_start)
> >>> +FUNC(__bp_harden_hyp_vecs_start)
> >>>   .rept 4
> >>>   vectors hyp_traps_vector
> >>>   .endr
> >>> -ENTRY(__bp_harden_hyp_vecs_end)
> >>> +GLOBAL(__bp_harden_hyp_vecs_end)
> >>> +END(__bp_harden_hyp_vecs_start)
> >>
> >> Shouldn't GLOBAL be changed to FUNC as well?
> >>
> >
> > I was a bit unsure but went for GLOBAL since the _end labels point to
> > addresses after and outside of the code sequence.
> > But I don't have a strong opinion and am happy to change them to FUNC
> > if you feel that's better.
>
> I don't think it should be FUNC as this is not meant to be called
> directly. I am also under the impression, we were planning to get rid of
> GLOBAL() as well.
>
> Furthermore, __bp_harden_hyp_vec_start is not a function per say. It is
> a pointer to the vector table.
>
>  From the brief look, the same remarks would apply to the rest of bpi.S.
> So I think we want to switch all the ENTRY() to LABEL().

Hi Julien,

The reason I choose FUNC for the start of the symbol is because these
symbols contain
executable code (not only a table of pointers to code somewhere else)
and the ELF spec
says that STT_FUNC means the symbol contains functions or other executable
code (not only callable functions IIUC):

"STT_FUNC The symbol is associated with a function or other executable code."
https://refspecs.linuxbase.org/elf/elf.pdf
(Symbol Table 1-20).

I think using LABEL instead of GLOBAL for the _end labels of these
code sequences makes sense.
I'm happy to change the _start labels to LABEL too if you guys feel
that's better.

Cheers,
Edgar

Re: [PATCH v4 14/17] xen: Add xen_mr_is_memory()

[Edgar E. Iglesias]

On Mon, May 6, 2024 at 11:59 AM Philippe Mathieu-Daudé
 wrote:
>
> On 2/5/24 09:26, David Hildenbrand wrote:
> > On 30.04.24 18:49, Edgar E. Iglesias wrote:
> >> From: "Edgar E. Iglesias" 
> >>
> >> Add xen_mr_is_memory() to abstract away tests for the
> >> xen_memory MR.
> >>
> >> Signed-off-by: Edgar E. Iglesias 
> >> ---
> >
> > [...]
> >
> >>   #endif
> >> diff --git a/system/physmem.c b/system/physmem.c
> >> index ad7a8c7d95..1a5ffcba2a 100644
> >> --- a/system/physmem.c
> >> +++ b/system/physmem.c
> >> @@ -2227,7 +2227,7 @@ static void *qemu_ram_ptr_length(RAMBlock
> >> *block, ram_addr_t addr,
> >>* because we don't want to map the entire memory in QEMU.
> >>* In that case just map the requested area.
> >>*/
> >> -if (block->offset == 0) {
> >> +if (xen_mr_is_memory(block->mr)) {
> >>   return xen_map_cache(block->mr, addr, len, lock, lock,
> >>is_write);
> >>   }
> >
> > I'd have moved that into a separate patch, because this is not a simple
> > abstraction here.
>
> Yes please, maybe using Stefano review comment in the description.
>

Thanks, for v5 I've split out this particular change into a separate patch:

softmmu: Replace check for RAMBlock offset 0 with xen_mr_is_memory

For xen, when checking for the first RAM (xen_memory), use
xen_mr_is_memory() rather than checking for a RAMBlock with
offset 0.

All Xen machines create xen_memory first so this has no
functional change for existing machines.

Signed-off-by: Edgar E. Iglesias 

diff --git a/system/physmem.c b/system/physmem.c
index ad7a8c7d95..1a5ffcba2a 100644
--- a/system/physmem.c
+++ b/system/physmem.c
@@ -2227,7 +2227,7 @@ static void *qemu_ram_ptr_length(RAMBlock
*block, ram_addr_t addr,
  * because we don't want to map the entire memory in QEMU.
  * In that case just map the requested area.
  */
-if (block->offset == 0) {
+if (xen_mr_is_memory(block->mr)) {
 return xen_map_cache(block->mr, addr, len, lock, lock,
  is_write);
 }



> >
> > Acked-by: David Hildenbrand 
> >
>

Re: [PATCH v3 6/9] xen/arm64: bpi: Add missing code symbol annotations

[Edgar E. Iglesias]

On Sat, May 4, 2024 at 2:14 AM Stefano Stabellini
 wrote:
>
> On Wed, 1 May 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > Use the generic xen/linkage.h macros to annotate code symbols
> > and add missing annotations.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  xen/arch/arm/arm64/bpi.S | 20 
> >  1 file changed, 12 insertions(+), 8 deletions(-)
> >
> > diff --git a/xen/arch/arm/arm64/bpi.S b/xen/arch/arm/arm64/bpi.S
> > index 4e63825220..b16e4d1e29 100644
> > --- a/xen/arch/arm/arm64/bpi.S
> > +++ b/xen/arch/arm/arm64/bpi.S
> > @@ -52,14 +52,15 @@
> >   * micro-architectures in a system.
> >   */
> >  .align   11
> > -ENTRY(__bp_harden_hyp_vecs_start)
> > +FUNC(__bp_harden_hyp_vecs_start)
> >  .rept 4
> >  vectors hyp_traps_vector
> >  .endr
> > -ENTRY(__bp_harden_hyp_vecs_end)
> > +GLOBAL(__bp_harden_hyp_vecs_end)
> > +END(__bp_harden_hyp_vecs_start)
>
> Shouldn't GLOBAL be changed to FUNC as well?
>

I was a bit unsure but went for GLOBAL since the _end labels point to
addresses after and outside of the code sequence.
But I don't have a strong opinion and am happy to change them to FUNC
if you feel that's better.

Cheers,
Edgar


>
> >  .macro mitigate_spectre_bhb_loop count
> > -ENTRY(__mitigate_spectre_bhb_loop_start_\count)
> > +FUNC(__mitigate_spectre_bhb_loop_start_\count)
> >  stp x0, x1, [sp, #-16]!
> >  mov x0, \count
> >  .Lspectre_bhb_loop\@:
> > @@ -68,11 +69,12 @@ ENTRY(__mitigate_spectre_bhb_loop_start_\count)
> >  b.ne.Lspectre_bhb_loop\@
> >  sb
> >  ldp x0, x1, [sp], #16
> > -ENTRY(__mitigate_spectre_bhb_loop_end_\count)
> > +GLOBAL(__mitigate_spectre_bhb_loop_end_\count)
>
> Also here?
>
>
> > +END(__mitigate_spectre_bhb_loop_start_\count)
> >  .endm
> >
> >  .macro smccc_workaround num smcc_id
> > -ENTRY(__smccc_workaround_smc_start_\num)
> > +FUNC(__smccc_workaround_smc_start_\num)
> >  sub sp, sp, #(8 * 4)
> >  stp x0, x1, [sp, #(8 * 2)]
> >  stp x2, x3, [sp, #(8 * 0)]
> > @@ -81,13 +83,15 @@ ENTRY(__smccc_workaround_smc_start_\num)
> >  ldp x2, x3, [sp, #(8 * 0)]
> >  ldp x0, x1, [sp, #(8 * 2)]
> >  add sp, sp, #(8 * 4)
> > -ENTRY(__smccc_workaround_smc_end_\num)
> > +GLOBAL(__smccc_workaround_smc_end_\num)
>
> And here?
>
>
> > +END(__smccc_workaround_smc_start_\num)
> >  .endm
> >
> > -ENTRY(__mitigate_spectre_bhb_clear_insn_start)
> > +FUNC(__mitigate_spectre_bhb_clear_insn_start)
> >  clearbhb
> >  isb
> > -ENTRY(__mitigate_spectre_bhb_clear_insn_end)
> > +GLOBAL(__mitigate_spectre_bhb_clear_insn_end)
>
> and here?
>
>
> > +END(__mitigate_spectre_bhb_clear_insn_start)
> >
> >  mitigate_spectre_bhb_loop 8
> >  mitigate_spectre_bhb_loop 24
> > --
> > 2.40.1
> >

Re: [PATCH v3 4/9] xen/arm64: head: Add missing code symbol annotations

[Edgar E. Iglesias]

On Sat, May 4, 2024 at 1:56 AM Stefano Stabellini
 wrote:
>
> On Wed, 1 May 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > Use the generic xen/linkage.h macros to annotate code symbols
> > and add missing annotations.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  xen/arch/arm/arm64/head.S | 50 ---
> >  1 file changed, 26 insertions(+), 24 deletions(-)
> >
> > diff --git a/xen/arch/arm/arm64/head.S b/xen/arch/arm/arm64/head.S
> > index fb297e9eb5..7acedb4f8f 100644
> > --- a/xen/arch/arm/arm64/head.S
> > +++ b/xen/arch/arm/arm64/head.S
> > @@ -90,7 +90,7 @@
> >   * 4K-aligned address.
> >   */
> >
> > -GLOBAL(start)
> > +FUNC(start)
> >  /*
> >   * DO NOT MODIFY. Image header expected by Linux boot-loaders.
> >   */
> > @@ -102,6 +102,7 @@ efi_head:
> >   */
> >  add x13, x18, #0x16
> >  b   real_start   /* branch to kernel start */
> > +END(start)
> >  .quad   0/* Image load offset from start of 
> > RAM */
> >  .quad   _end - start /* Effective size of kernel image, 
> > little-endian */
> >  .quad   __HEAD_FLAGS /* Informative flags, little-endian */
> > @@ -223,7 +224,7 @@ section_table:
> >  .align  5
> >  #endif /* CONFIG_ARM_EFI */
> >
> > -real_start:
> > +FUNC_LOCAL(real_start)
> >  /* BSS should be zeroed when booting without EFI */
> >  mov   x26, #0/* x26 := skip_zero_bss */
> >
> > @@ -263,9 +264,9 @@ primary_switched:
> >  mov   x1, x21/* x1 := paddr(FDT) */
> >  ldr   x2, =start_xen
> >  b launch
> > -ENDPROC(real_start)
> > +END(real_start)
> >
> > -GLOBAL(init_secondary)
> > +FUNC(init_secondary)
> >  msr   DAIFSet, 0xf   /* Disable all interrupts */
> >
> >  /* Find out where we are */
> > @@ -304,7 +305,7 @@ secondary_switched:
> >  /* Jump to C world */
> >  ldr   x2, =start_secondary
> >  b launch
> > -ENDPROC(init_secondary)
> > +END(init_secondary)
> >
> >  /*
> >   * Check if the CPU has been booted in Hypervisor mode.
> > @@ -313,7 +314,7 @@ ENDPROC(init_secondary)
> >   *
> >   * Clobbers x0 - x5
> >   */
> > -check_cpu_mode:
> > +FUNC_LOCAL(check_cpu_mode)
> >  PRINT_ID("- Current EL ")
> >  mrs   x5, CurrentEL
> >  print_reg x5
> > @@ -329,7 +330,7 @@ check_cpu_mode:
> >  PRINT_ID("- Xen must be entered in NS EL2 mode -\r\n")
> >  PRINT_ID("- Please update the bootloader -\r\n")
> >  b fail
> > -ENDPROC(check_cpu_mode)
> > +END(check_cpu_mode)
> >
> >  /*
> >   * Zero BSS
> > @@ -339,7 +340,7 @@ ENDPROC(check_cpu_mode)
> >   *
> >   * Clobbers x0 - x3
> >   */
> > -zero_bss:
> > +FUNC_LOCAL(zero_bss)
> >  /* Zero BSS only when requested */
> >  cbnz  x26, skip_bss
> >
> > @@ -353,14 +354,14 @@ zero_bss:
> >
> >  skip_bss:
> >  ret
> > -ENDPROC(zero_bss)
> > +END(zero_bss)
> >
> >  /*
> >   * Initialize the processor for turning the MMU on.
> >   *
> >   * Clobbers x0 - x3
> >   */
> > -cpu_init:
> > +FUNC_LOCAL(cpu_init)
> >  PRINT_ID("- Initialize CPU -\r\n")
> >
> >  /* Set up memory attribute type tables */
> > @@ -399,7 +400,7 @@ cpu_init:
> >   */
> >  msr spsel, #1
> >  ret
> > -ENDPROC(cpu_init)
> > +END(cpu_init)
> >
> >  /*
> >   * Setup the initial stack and jump to the C world
> > @@ -411,7 +412,7 @@ ENDPROC(cpu_init)
> >   *
> >   * Clobbers x3
> >   */
> > -launch:
> > +FUNC_LOCAL(launch)
> >  ldr   x3, =init_data
> >  add   x3, x3, #INITINFO_stack /* Find the boot-time stack */
> >  ldr   x3, [x3]
> > @@ -421,13 +422,13 @@ launch:
> >
> >  /* Jump to C world */
> >  brx2
> > -ENDPROC(launch)
> > +END(launch)
> >
> >  /* Fail-stop */
> > -fail:   PRINT_ID("- Boot failed -\r\n")
> > +FUNC_LOCAL(fail)   PRINT_ID("- Boot failed -\r\n")
>
> Maybe we should move PRINT_ID to a newline?
> I am not sure FUN

Re: [PATCH v4 16/17] xen: mapcache: Add support for grant mappings

[Edgar E. Iglesias]

On Thu, May 2, 2024 at 9:18 PM Stefano Stabellini
 wrote:
>
> On Tue, 30 Apr 2024, Edgar E. Iglesias wrote:
> > From: "Edgar E. Iglesias" 
> >
> > Add a second mapcache for grant mappings. The mapcache for
> > grants needs to work with XC_PAGE_SIZE granularity since
> > we can't map larger ranges than what has been granted to us.
> >
> > Like with foreign mappings (xen_memory), machines using grants
> > are expected to initialize the xen_grants MR and map it
> > into their address-map accordingly.
> >
> > Signed-off-by: Edgar E. Iglesias 
> > ---
> >  hw/xen/xen-hvm-common.c |  12 ++-
> >  hw/xen/xen-mapcache.c   | 158 +---
> >  include/hw/xen/xen-hvm-common.h |   3 +
> >  include/sysemu/xen.h|   7 ++
> >  4 files changed, 145 insertions(+), 35 deletions(-)
> >
> > diff --git a/hw/xen/xen-hvm-common.c b/hw/xen/xen-hvm-common.c
> > index 0267b88d26..fdec400491 100644
> > --- a/hw/xen/xen-hvm-common.c
> > +++ b/hw/xen/xen-hvm-common.c
> > @@ -10,12 +10,18 @@
> >  #include "hw/boards.h"
> >  #include "hw/xen/arch_hvm.h"
> >
> > -MemoryRegion xen_memory;
> > +MemoryRegion xen_memory, xen_grants;
> >
> > -/* Check for xen memory.  */
> > +/* Check for any kind of xen memory, foreign mappings or grants.  */
> >  bool xen_mr_is_memory(MemoryRegion *mr)
> >  {
> > -return mr == &xen_memory;
> > +return mr == &xen_memory || mr == &xen_grants;
> > +}
> > +
> > +/* Check specifically for grants.  */
> > +bool xen_mr_is_grants(MemoryRegion *mr)
> > +{
> > +return mr == &xen_grants;
> >  }
> >
> >  void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
> > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > index 1b32d0c003..96cd68e28d 100644
> > --- a/hw/xen/xen-mapcache.c
> > +++ b/hw/xen/xen-mapcache.c
> > @@ -14,6 +14,7 @@
> >
> >  #include 
> >
> > +#include "hw/xen/xen-hvm-common.h"
> >  #include "hw/xen/xen_native.h"
> >  #include "qemu/bitmap.h"
> >
> > @@ -21,6 +22,8 @@
> >  #include "sysemu/xen-mapcache.h"
> >  #include "trace.h"
> >
> > +#include 
> > +#include 
> >
> >  #if HOST_LONG_BITS == 32
> >  #  define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
> > @@ -41,6 +44,7 @@ typedef struct MapCacheEntry {
> >  unsigned long *valid_mapping;
> >  uint32_t lock;
> >  #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
> > +#define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
> >  uint8_t flags;
> >  hwaddr size;
> >
> > @@ -74,6 +78,8 @@ typedef struct MapCache {
> >  } MapCache;
> >
> >  static MapCache *mapcache;
> > +static MapCache *mapcache_grants;
> > +static xengnttab_handle *xen_region_gnttabdev;
> >
> >  static inline void mapcache_lock(MapCache *mc)
> >  {
> > @@ -132,6 +138,12 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
> > *opaque)
> >  unsigned long max_mcache_size;
> >  unsigned int bucket_shift;
> >
> > +xen_region_gnttabdev = xengnttab_open(NULL, 0);
> > +if (xen_region_gnttabdev == NULL) {
> > +error_report("mapcache: Failed to open gnttab device");
> > +exit(EXIT_FAILURE);
> > +}
> > +
> >  if (HOST_LONG_BITS == 32) {
> >  bucket_shift = 16;
> >  } else {
> > @@ -160,6 +172,15 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void 
> > *opaque)
> >  mapcache = xen_map_cache_init_single(f, opaque,
> >   bucket_shift,
> >   max_mcache_size);
> > +
> > +/*
> > + * Grant mappings must use XC_PAGE_SIZE granularity since we can't
> > + * map anything beyond the number of pages granted to us.
> > + */
> > +mapcache_grants = xen_map_cache_init_single(f, opaque,
> > +XC_PAGE_SHIFT,
> > +max_mcache_size);
> > +
> >  setrlimit(RLIMIT_AS, &rlimit_as);
> >  }
> >
> > @@ -169,17 +190,25 @@ static void xen_remap_bucket(MapCache *mc,
> >   hwaddr size,
> >   hwaddr address_index,
> >   bool dummy,
> > + 

Re: [PATCH v4 15/17] xen: mapcache: Remove assumption of RAMBlock with 0 offset

[Edgar E. Iglesias]

On Thu, May 2, 2024 at 8:53 PM Stefano Stabellini
 wrote:
>
> +Xenia
>
> On Thu, 2 May 2024, Edgar E. Iglesias wrote:
> > On Wed, May 1, 2024 at 11:24 PM Stefano Stabellini
> >  wrote:
> > >
> > > On Tue, 30 Apr 2024, Edgar E. Iglesias wrote:
> > > > From: "Edgar E. Iglesias" 
> > > >
> > > > The current mapcache assumes that all memory is mapped
> > > > in a single RAM MR (the first one with offset 0). Remove
> > > > this assumption and propagate the offset to the mapcache
> > > > so it can do reverse mappings (from hostptr -> ram_addr).
> > > >
> > > > This is in preparation for adding grant mappings.
> > > >
> > > > Signed-off-by: Edgar E. Iglesias 
> > >
> > >
> > > Looking at xen_remap_bucket, it is only using address_index (without
> > > adding ram_offset) to map foreign memory. From xen_remap_bucket, I would
> > > understand that address_index already includes the ram_offset.
> > >
> > > Meaning that if we want to map foreign mapping at address 0x5000, then
> > > address_index would be 0x5000, even if ram_offset is 0x1000.
> > >
> > > But then looking xen_ram_addr_from_mapcache_single ram_offset is added
> > > to paddr_index to calculate the physical address. So in that case we
> > > would want address_index to be 0x4000 and ram_offset to be 0x1000. But
> > > xen_remap_bucket would have to sum address_index and ram_offset to map
> > > foreign memory.
> > >
> > > So I am a bit confused, did I get it wrong? One more comment below.
> > >
> >
> > Thanks Stefano,
> >
> > I think the confusion is that this ram_addr_offset is not related to
> > guest address-space.
> > It's a QEMU internal thing and it shouldn't be included in the address
> > used to map foreign memory.
> > The mapcache can treat this ram_addr offset like a cookie that we keep
> > around to be able to do
> > reverse mappings from host pointers into ram_addr space
> > (xen_ram_addr_from_mapcache).
> >
> > The current mapcache implementation works because we've really only
> > been using foreign mappings
> > on RAMBlocks with offset 0. We're also creating RAM's such that the
> > offset into the RAM is also
> > the guest physical address, for x86 this is natural since RAM starts
> > at zero (for lowmem) but for
> > ARM we're creating larger than needed RAM's (GUEST_RAM0_BASE + ram-size) to
> > make this assumption true. Anyway, In this series I'm not addressing
> > this second assumption.
>
> Let's see if I understand correctly.
>
> The ram_addr space is an internal QEMU address space which is different
> from the guest physical address space and thus cannot and should not be
> used to do foreign mappings (foreign mapping hypercalls take a guest
> physical or a real physical address to map). Is that correct?
>
> If so, then I understand.
>

Yes, that matches my understanding.

>
>
> > There's a second call in physmem.c to xen_map_cache using the
> > block->offset as an address.
> > I was considering removing that second call since I can't see how it can 
> > work
> > (except perhaps in some specific use-case by luck?). Anyway, for now
> > I've left it unmodified.
>
> Yes, that code was written with the assumption that block->offset is an
> offset in the guest physical address space and could be used as a guest
> physical address. Actually, you might have spotted a real bug.
>
> The intent was for smaller regions (not the bit RAM region, things like
> a ROM region for instance) we could map them in full. So here we were
> trying to map the whole thing from start to finish using block->offset
> as start.
>
>
> > > > ---
> > > >  hw/xen/xen-mapcache.c | 25 ++---
> > > >  include/sysemu/xen-mapcache.h |  2 ++
> > > >  system/physmem.c  |  8 
> > > >  3 files changed, 24 insertions(+), 11 deletions(-)
> > > >
> > > > diff --git a/hw/xen/xen-mapcache.c b/hw/xen/xen-mapcache.c
> > > > index 09b5f36d9c..1b32d0c003 100644
> > > > --- a/hw/xen/xen-mapcache.c
> > > > +++ b/hw/xen/xen-mapcache.c
> > > > @@ -43,6 +43,9 @@ typedef struct MapCacheEntry {
> > > >  #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
> > > >  uint8_t flags;
> > > >  hwaddr size;
> > > > +
> > > > 

[PATCH v3 5/9] xen/arm64: debug: Add missing code symbol annotations

[Edgar E. Iglesias]

From: "Edgar E. Iglesias" 

Use the generic xen/linkage.h macros to annotate code symbols
and add missing annotations.

Signed-off-by: Edgar E. Iglesias 
---
 xen/arch/arm/arm64/debug.S | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/arm64/debug.S b/xen/arch/arm/arm64/debug.S
index 71cad9d762..c3d02c33d7 100644
--- a/xen/arch/arm/arm64/debug.S
+++ b/xen/arch/arm/arm64/debug.S
@@ -27,17 +27,19 @@
  * Print a character on the UART - this function is called by C
  * x0: character to print
  */
-GLOBAL(early_putch)
+FUNC(early_putch)
 ldr   x15, =EARLY_UART_VIRTUAL_ADDRESS
 early_uart_ready x15, 1
 early_uart_transmit x15, w0
 ret
+END(early_putch)
 
 /* Flush the UART - this function is called by C */
-GLOBAL(early_flush)
+FUNC(early_flush)
 ldr   x15, =EARLY_UART_VIRTUAL_ADDRESS  /* x15 := VA UART base address 
*/
 early_uart_ready x15, 1
 ret
+END(early_flush)
 
 /*
  * Local variables:
-- 
2.40.1