Re: [PATCH v8 13/19] hvf: Add Apple Silicon support
Hi Peter,
On 15.06.21 12:21, Peter Maydell wrote:
> On Wed, 19 May 2021 at 21:23, Alexander Graf wrote:
>> With Apple Silicon available to the masses, it's a good time to add support
>> for driving its virtualization extensions from QEMU.
>>
>> This patch adds all necessary architecture specific code to get basic VMs
>> working. It's still pretty raw, but definitely functional.
>>
>> Known limitations:
>>
>> - Vtimer acknowledgement is hacky
>> - Should implement more sysregs and fault on invalid ones then
>> - WFI handling is missing, need to marry it with vtimer
>>
>> Signed-off-by: Alexander Graf
>> Reviewed-by: Roman Bolshakov
>> @@ -446,11 +454,17 @@ static void hvf_start_vcpu_thread(CPUState *cpu)
>> cpu, QEMU_THREAD_JOINABLE);
>> }
>>
>> +__attribute__((weak)) void hvf_kick_vcpu_thread(CPUState *cpu)
>> +{
>> +cpus_kick_thread(cpu);
>> +}
> Why is this marked 'weak' ? If there's a reason for it then
> it ought to have a comment describing the reason. If we can avoid
> it then we should do so -- past experience is that 'weak' refs
> are rather non-portable, though at least this one is in a
> host-OS-specific file.
Mostly because I wanted to keep the kick function in the generic file
for the generic case. ARM is special in that it requires different kick
mechanisms depending on which context we're in (in-vcpu or in-QEMU).
However, I agree that with 2 architectures, there's not really a
"default". I'm happy to move it into the x86 specific file.
>
>> +static void hvf_raise_exception(CPUARMState *env, uint32_t excp,
>> +uint32_t syndrome)
>> +{
>> +unsigned int new_el = 1;
>> +unsigned int old_mode = pstate_read(env);
>> +unsigned int new_mode = aarch64_pstate_mode(new_el, true);
>> +target_ulong addr = env->cp15.vbar_el[new_el];
>> +
>> +env->cp15.esr_el[new_el] = syndrome;
>> +aarch64_save_sp(env, arm_current_el(env));
>> +env->elr_el[new_el] = env->pc;
>> +env->banked_spsr[aarch64_banked_spsr_index(new_el)] = old_mode;
>> +pstate_write(env, PSTATE_DAIF | new_mode);
>> +aarch64_restore_sp(env, new_el);
>> +env->pc = addr;
>> +}
> KVM does "raise an exception" by calling arm_cpu_do_interrupt()
> to do the "set ESR_ELx, save SPSR, etc etc" work (see eg
> kvm_arm_handle_debug()". Does that not work here ?
It works like a charm. I mostly did the dance because I was under the
impression you wanted to avoid me calling into any TCG code. And to me
arm_cpu_do_interrupt() seemed like TCG code. I'm absolutely happy to
change it though. Leaving things to generic code is good IMHO :).
>
>> +
>> +static uint64_t hvf_sysreg_read(CPUState *cpu, uint32_t reg)
>> +{
>> +ARMCPU *arm_cpu = ARM_CPU(cpu);
>> +uint64_t val = 0;
>> +
>> +switch (reg) {
>> +case SYSREG_CNTPCT_EL0:
>> +val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
>> + gt_cntfrq_period_ns(arm_cpu);
>> +break;
> Does hvf handle the "EL0 access which should be denied because
> CNTKCTL_EL1.EL0PCTEN is 0" case for us, or should we have
> an access-check here ?
A quick test where I tried to access it in a VM in EL0 shows that either
the CPU or HVF already generates the trap. So no check needed.
>
>> +case SYSREG_PMCCNTR_EL0:
>> +val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
> This is supposed to be a cycle counter, not a timestamp...
At 1Ghz cycle frequency, what's the difference? Or are you concerned
about the lack of overflow and PMCR logic?
>
>> +break;
>> +default:
>> +trace_hvf_unhandled_sysreg_read(reg,
>> +(reg >> 20) & 0x3,
>> +(reg >> 14) & 0x7,
>> +(reg >> 10) & 0xf,
>> +(reg >> 1) & 0xf,
>> +(reg >> 17) & 0x7);
>> +break;
>> +}
>> +
>> +return val;
>> +}
>> +
>> +static void hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val)
>> +{
>> +switch (reg) {
>> +case SYSREG_CNTPCT_EL0:
>> +break;
> CNTPCT_EL0 is read-only (ie writes should fault) but this
> makes it writes-ignored, doesn't it ?
It does indeed, let me fix that.
>
>> +default:
>> +trace_hvf_unhandled_sysreg_write(reg,
>> + (reg >> 20) & 0x3,
>> + (reg >> 14) & 0x7,
>> + (reg >> 10) & 0xf,
>> + (reg >> 1) & 0xf,
>> + (reg >> 17) & 0x7);
>> +break;
>> +}
>> +}
>> +switch (ec) {
>> +case EC_DATAABORT: {
>> +bool isv = syndrome & ARM_EL_ISV;
>> +bool iswrite = (syndrome >> 6) & 1;
>> +bool s1ptw = (syndrome >> 7) & 1;
>> +uint32_t sas = (syndrome >> 22) & 3;
>> +uint32_t len = 1 << sas;
>>
Re: [PATCH v8 13/19] hvf: Add Apple Silicon support
On Wed, 19 May 2021 at 21:23, Alexander Graf wrote:
>
> With Apple Silicon available to the masses, it's a good time to add support
> for driving its virtualization extensions from QEMU.
>
> This patch adds all necessary architecture specific code to get basic VMs
> working. It's still pretty raw, but definitely functional.
>
> Known limitations:
>
> - Vtimer acknowledgement is hacky
> - Should implement more sysregs and fault on invalid ones then
> - WFI handling is missing, need to marry it with vtimer
>
> Signed-off-by: Alexander Graf
> Reviewed-by: Roman Bolshakov
> @@ -446,11 +454,17 @@ static void hvf_start_vcpu_thread(CPUState *cpu)
> cpu, QEMU_THREAD_JOINABLE);
> }
>
> +__attribute__((weak)) void hvf_kick_vcpu_thread(CPUState *cpu)
> +{
> +cpus_kick_thread(cpu);
> +}
Why is this marked 'weak' ? If there's a reason for it then
it ought to have a comment describing the reason. If we can avoid
it then we should do so -- past experience is that 'weak' refs
are rather non-portable, though at least this one is in a
host-OS-specific file.
> +static void hvf_raise_exception(CPUARMState *env, uint32_t excp,
> +uint32_t syndrome)
> +{
> +unsigned int new_el = 1;
> +unsigned int old_mode = pstate_read(env);
> +unsigned int new_mode = aarch64_pstate_mode(new_el, true);
> +target_ulong addr = env->cp15.vbar_el[new_el];
> +
> +env->cp15.esr_el[new_el] = syndrome;
> +aarch64_save_sp(env, arm_current_el(env));
> +env->elr_el[new_el] = env->pc;
> +env->banked_spsr[aarch64_banked_spsr_index(new_el)] = old_mode;
> +pstate_write(env, PSTATE_DAIF | new_mode);
> +aarch64_restore_sp(env, new_el);
> +env->pc = addr;
> +}
KVM does "raise an exception" by calling arm_cpu_do_interrupt()
to do the "set ESR_ELx, save SPSR, etc etc" work (see eg
kvm_arm_handle_debug()". Does that not work here ?
> +
> +static uint64_t hvf_sysreg_read(CPUState *cpu, uint32_t reg)
> +{
> +ARMCPU *arm_cpu = ARM_CPU(cpu);
> +uint64_t val = 0;
> +
> +switch (reg) {
> +case SYSREG_CNTPCT_EL0:
> +val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
> + gt_cntfrq_period_ns(arm_cpu);
> +break;
Does hvf handle the "EL0 access which should be denied because
CNTKCTL_EL1.EL0PCTEN is 0" case for us, or should we have
an access-check here ?
> +case SYSREG_PMCCNTR_EL0:
> +val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
This is supposed to be a cycle counter, not a timestamp...
> +break;
> +default:
> +trace_hvf_unhandled_sysreg_read(reg,
> +(reg >> 20) & 0x3,
> +(reg >> 14) & 0x7,
> +(reg >> 10) & 0xf,
> +(reg >> 1) & 0xf,
> +(reg >> 17) & 0x7);
> +break;
> +}
> +
> +return val;
> +}
> +
> +static void hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val)
> +{
> +switch (reg) {
> +case SYSREG_CNTPCT_EL0:
> +break;
CNTPCT_EL0 is read-only (ie writes should fault) but this
makes it writes-ignored, doesn't it ?
> +default:
> +trace_hvf_unhandled_sysreg_write(reg,
> + (reg >> 20) & 0x3,
> + (reg >> 14) & 0x7,
> + (reg >> 10) & 0xf,
> + (reg >> 1) & 0xf,
> + (reg >> 17) & 0x7);
> +break;
> +}
> +}
> +switch (ec) {
> +case EC_DATAABORT: {
> +bool isv = syndrome & ARM_EL_ISV;
> +bool iswrite = (syndrome >> 6) & 1;
> +bool s1ptw = (syndrome >> 7) & 1;
> +uint32_t sas = (syndrome >> 22) & 3;
> +uint32_t len = 1 << sas;
> +uint32_t srt = (syndrome >> 16) & 0x1f;
> +uint64_t val = 0;
> +
> +trace_hvf_data_abort(env->pc, hvf_exit->exception.virtual_address,
> + hvf_exit->exception.physical_address, isv,
> + iswrite, s1ptw, len, srt);
> +
> +assert(isv);
This seems dubious -- won't we just crash if the guest does
a data access to a device or to unmapped memory with an insn that
doesn't set ISV ? With KVM we feed this back to the guest as an
external data abort (see the KVM_EXIT_ARM_NISV handling).
> +
> +if (iswrite) {
> +val = hvf_get_reg(cpu, srt);
> +address_space_write(_space_memory,
> +hvf_exit->exception.physical_address,
> +MEMTXATTRS_UNSPECIFIED, , len);
> +} else {
> +address_space_read(_space_memory,
> + hvf_exit->exception.physical_address,
> + MEMTXATTRS_UNSPECIFIED, , len);
> +hvf_set_reg(cpu, srt,
Re: [PATCH v8 13/19] hvf: Add Apple Silicon support
On Wed, May 19, 2021 at 10:22:47PM +0200, Alexander Graf wrote: > With Apple Silicon available to the masses, it's a good time to add support > for driving its virtualization extensions from QEMU. > > This patch adds all necessary architecture specific code to get basic VMs > working. It's still pretty raw, but definitely functional. > > Known limitations: > > - Vtimer acknowledgement is hacky > - Should implement more sysregs and fault on invalid ones then > - WFI handling is missing, need to marry it with vtimer > > Signed-off-by: Alexander Graf > Reviewed-by: Roman Bolshakov > > --- > > v1 -> v2: > > - Merge vcpu kick function patch > - Implement WFI handling (allows vCPUs to sleep) > - Synchronize system registers (fixes OVMF crashes and reboot) > - Don't always call cpu_synchronize_state() > - Use more fine grained iothread locking > - Populate aa64mmfr0 from hardware > > v2 -> v3: > > - Advance PC on SMC > - Use cp list interface for sysreg syncs > - Do not set current_cpu > - Fix sysreg isread mask > - Move sysreg handling to functions > - Remove WFI logic again > - Revert to global iothread locking > - Use Hypervisor.h on arm, hv.h does not contain aarch64 definitions > > v3 -> v4: > > - No longer include Hypervisor.h > > v5 -> v6: > > - Swap sysreg definition order. This way we're in line with asm outputs. > > v6 -> v7: > > - Remove osdep.h include from hvf_int.h > - Synchronize SIMD registers as well > - Prepend 0x for hex values > - Convert DPRINTF to trace points > - Use main event loop (fixes gdbstub issues) > - Remove PSCI support, inject UDEF on HVC/SMC > - Change vtimer logic to look at ctl.istatus for vtimer mask sync > - Add kick callback again (fixes remote CPU notification) > > v7 -> v8: > > - Fix checkpatch errors > --- > MAINTAINERS | 5 + > accel/hvf/hvf-accel-ops.c | 14 + > include/sysemu/hvf_int.h| 9 +- > meson.build | 1 + > target/arm/hvf/hvf.c| 703 > target/arm/hvf/trace-events | 10 + > 6 files changed, 741 insertions(+), 1 deletion(-) > create mode 100644 target/arm/hvf/hvf.c > create mode 100644 target/arm/hvf/trace-events Reviewed-by: Sergio Lopez Tested-by: Sergio Lopez signature.asc Description: PGP signature
[PATCH v8 13/19] hvf: Add Apple Silicon support
With Apple Silicon available to the masses, it's a good time to add support
for driving its virtualization extensions from QEMU.
This patch adds all necessary architecture specific code to get basic VMs
working. It's still pretty raw, but definitely functional.
Known limitations:
- Vtimer acknowledgement is hacky
- Should implement more sysregs and fault on invalid ones then
- WFI handling is missing, need to marry it with vtimer
Signed-off-by: Alexander Graf
Reviewed-by: Roman Bolshakov
---
v1 -> v2:
- Merge vcpu kick function patch
- Implement WFI handling (allows vCPUs to sleep)
- Synchronize system registers (fixes OVMF crashes and reboot)
- Don't always call cpu_synchronize_state()
- Use more fine grained iothread locking
- Populate aa64mmfr0 from hardware
v2 -> v3:
- Advance PC on SMC
- Use cp list interface for sysreg syncs
- Do not set current_cpu
- Fix sysreg isread mask
- Move sysreg handling to functions
- Remove WFI logic again
- Revert to global iothread locking
- Use Hypervisor.h on arm, hv.h does not contain aarch64 definitions
v3 -> v4:
- No longer include Hypervisor.h
v5 -> v6:
- Swap sysreg definition order. This way we're in line with asm outputs.
v6 -> v7:
- Remove osdep.h include from hvf_int.h
- Synchronize SIMD registers as well
- Prepend 0x for hex values
- Convert DPRINTF to trace points
- Use main event loop (fixes gdbstub issues)
- Remove PSCI support, inject UDEF on HVC/SMC
- Change vtimer logic to look at ctl.istatus for vtimer mask sync
- Add kick callback again (fixes remote CPU notification)
v7 -> v8:
- Fix checkpatch errors
---
MAINTAINERS | 5 +
accel/hvf/hvf-accel-ops.c | 14 +
include/sysemu/hvf_int.h| 9 +-
meson.build | 1 +
target/arm/hvf/hvf.c| 703
target/arm/hvf/trace-events | 10 +
6 files changed, 741 insertions(+), 1 deletion(-)
create mode 100644 target/arm/hvf/hvf.c
create mode 100644 target/arm/hvf/trace-events
diff --git a/MAINTAINERS b/MAINTAINERS
index 262e96714b..f3b4fdcf60 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -428,6 +428,11 @@ F: accel/accel-*.c
F: accel/Makefile.objs
F: accel/stubs/Makefile.objs
+Apple Silicon HVF CPUs
+M: Alexander Graf
+S: Maintained
+F: target/arm/hvf/
+
X86 HVF CPUs
M: Cameron Esfahani
M: Roman Bolshakov
diff --git a/accel/hvf/hvf-accel-ops.c b/accel/hvf/hvf-accel-ops.c
index d1691be989..48e402ef57 100644
--- a/accel/hvf/hvf-accel-ops.c
+++ b/accel/hvf/hvf-accel-ops.c
@@ -60,6 +60,10 @@
HVFState *hvf_state;
+#ifdef __aarch64__
+#define HV_VM_DEFAULT NULL
+#endif
+
/* Memory slots */
hvf_slot *hvf_find_overlap_slot(uint64_t start, uint64_t size)
@@ -375,7 +379,11 @@ static int hvf_init_vcpu(CPUState *cpu)
pthread_sigmask(SIG_BLOCK, NULL, );
sigdelset(, SIG_IPI);
+#ifdef __aarch64__
+r = hv_vcpu_create(>hvf->fd, (hv_vcpu_exit_t **)>hvf->exit,
NULL);
+#else
r = hv_vcpu_create((hv_vcpuid_t *)>hvf->fd, HV_VCPU_DEFAULT);
+#endif
cpu->vcpu_dirty = 1;
assert_hvf_ok(r);
@@ -446,11 +454,17 @@ static void hvf_start_vcpu_thread(CPUState *cpu)
cpu, QEMU_THREAD_JOINABLE);
}
+__attribute__((weak)) void hvf_kick_vcpu_thread(CPUState *cpu)
+{
+cpus_kick_thread(cpu);
+}
+
static void hvf_accel_ops_class_init(ObjectClass *oc, void *data)
{
AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);
ops->create_vcpu_thread = hvf_start_vcpu_thread;
+ops->kick_vcpu_thread = hvf_kick_vcpu_thread;
ops->synchronize_post_reset = hvf_cpu_synchronize_post_reset;
ops->synchronize_post_init = hvf_cpu_synchronize_post_init;
diff --git a/include/sysemu/hvf_int.h b/include/sysemu/hvf_int.h
index 8b66a4e7d0..e52d67ed5c 100644
--- a/include/sysemu/hvf_int.h
+++ b/include/sysemu/hvf_int.h
@@ -11,7 +11,11 @@
#ifndef HVF_INT_H
#define HVF_INT_H
+#ifdef __aarch64__
+#include
+#else
#include
+#endif
/* hvf_slot flags */
#define HVF_SLOT_LOG (1 << 0)
@@ -44,7 +48,9 @@ struct HVFState {
extern HVFState *hvf_state;
struct hvf_vcpu_state {
-int fd;
+uint64_t fd;
+void *exit;
+bool vtimer_masked;
};
void assert_hvf_ok(hv_return_t ret);
@@ -54,5 +60,6 @@ int hvf_vcpu_exec(CPUState *);
hvf_slot *hvf_find_overlap_slot(uint64_t, uint64_t);
int hvf_put_registers(CPUState *);
int hvf_get_registers(CPUState *);
+void hvf_kick_vcpu_thread(CPUState *cpu);
#endif
diff --git a/meson.build b/meson.build
index a58a75d056..698f4e9356 100644
--- a/meson.build
+++ b/meson.build
@@ -1856,6 +1856,7 @@ if have_system or have_user
'accel/tcg',
'hw/core',
'target/arm',
+'target/arm/hvf',
'target/hppa',
'target/i386',
'target/i386/kvm',
diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c
new file mode 100644
index 00..3934c05979
--- /dev/null
+++ b/target/arm/hvf/hvf.c
@@ -0,0 +1,703 @@
+/*
+ * QEMU Hypervisor.framework support for Apple
