Implement ARM64 architectural vCPU state preservation and retrieval handlers over LUO using struct kvm_vcpu_arch_ser in include/linux/kho/abi/kvm_arm64.h.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/arm64/kvm/Kconfig | 1 + arch/arm64/kvm/Makefile | 2 + arch/arm64/kvm/kvm_luo.c | 201 ++++++++++++++++++++++++++++++ include/linux/kho/abi/kvm_arm64.h | 61 +++++++++ 4 files changed, 265 insertions(+) create mode 100644 arch/arm64/kvm/kvm_luo.c create mode 100644 include/linux/kho/abi/kvm_arm64.h diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f9a485..fd25b3c0d8ca 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -37,6 +37,7 @@ menuconfig KVM select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS select KVM_GUEST_MEMFD + select HAVE_KVM_ARCH_VCPU_PRESERVE help Support hosting virtualized guest machines. diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 59612d2f277c..d5e888d1bd50 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -47,3 +47,5 @@ $(obj)/hyp_constants.h: $(obj)/hyp-constants.s FORCE obj-kvm := $(addprefix $(obj)/, $(kvm-y)) $(obj-kvm): $(obj)/hyp_constants.h + +kvm-y += kvm_luo.o diff --git a/arch/arm64/kvm/kvm_luo.c b/arch/arm64/kvm/kvm_luo.c new file mode 100644 index 000000000000..59c56836cda8 --- /dev/null +++ b/arch/arm64/kvm/kvm_luo.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + * + * ARM64 KVM LUO preservation and retrieval handlers. + */ + +#include <linux/kexec_handover.h> +#include <linux/kho/abi/kvm_arm64.h> +#include <linux/kvm_host.h> +#include <linux/sched.h> + +#include <asm/kvm_emulate.h> +#include <asm/kvm_mmu.h> + +#include <kvm/arm_arch_timer.h> +#include <kvm/arm_vgic.h> + +#include "sys_regs.h" +#include "vgic/vgic.h" + +int kvm_arch_vm_luo_preserve(struct kvm *kvm, struct kvm_luo_ser *ser) +{ + ser->type = kvm_phys_shift(&kvm->arch.mmu); + if (kvm_vm_is_protected(kvm)) + ser->type |= KVM_VM_TYPE_ARM_PROTECTED; + + return 0; +} + +int kvm_arch_vm_luo_retrieve(struct kvm *kvm, struct kvm_luo_ser *ser) +{ + return 0; +} + +void kvm_arch_vm_luo_unpreserve(struct kvm *kvm, struct kvm_luo_ser *ser) +{ +} + +void kvm_arch_vm_luo_finish(struct kvm_luo_ser *ser) +{ +} + +static void kvm_arm_luo_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + regs->regs = vcpu->arch.ctxt.regs; + regs->sp_el1 = ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); + regs->elr_el1 = ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); + regs->spsr[KVM_SPSR_EL1] = ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1); + regs->spsr[KVM_SPSR_ABT] = vcpu->arch.ctxt.spsr_abt; + regs->spsr[KVM_SPSR_UND] = vcpu->arch.ctxt.spsr_und; + regs->spsr[KVM_SPSR_IRQ] = vcpu->arch.ctxt.spsr_irq; + regs->spsr[KVM_SPSR_FIQ] = vcpu->arch.ctxt.spsr_fiq; + regs->fp_regs = vcpu->arch.ctxt.fp_regs; +} + +static void kvm_arm_luo_set_regs(struct kvm_vcpu *vcpu, const struct kvm_regs *regs) +{ + vcpu->arch.ctxt.regs = regs->regs; + ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1) = regs->sp_el1; + ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1) = regs->elr_el1; + ctxt_sys_reg(&vcpu->arch.ctxt, SPSR_EL1) = regs->spsr[KVM_SPSR_EL1]; + vcpu->arch.ctxt.spsr_abt = regs->spsr[KVM_SPSR_ABT]; + vcpu->arch.ctxt.spsr_und = regs->spsr[KVM_SPSR_UND]; + vcpu->arch.ctxt.spsr_irq = regs->spsr[KVM_SPSR_IRQ]; + vcpu->arch.ctxt.spsr_fiq = regs->spsr[KVM_SPSR_FIQ]; + vcpu->arch.ctxt.fp_regs = regs->fp_regs; +} + +int kvm_arch_vcpu_luo_preserve(struct kvm_vcpu *vcpu, struct kvm_vcpu_ser *ser) +{ + struct kvm_vcpu_arch_ser *state; + int num_sysregs; + u64 *indices; + size_t size; + int i; + + num_sysregs = kvm_arm_get_sys_reg_indices(vcpu, NULL); + indices = kmalloc_array(num_sysregs, sizeof(u64), GFP_KERNEL); + if (!indices) + return -ENOMEM; + num_sysregs = kvm_arm_get_sys_reg_indices(vcpu, indices); + + size = struct_size(state, sysregs, num_sysregs); + state = kho_alloc_preserve(size); + if (IS_ERR(state)) { + kfree(indices); + return PTR_ERR(state); + } + + /* Core register state (uAPI struct kvm_regs) */ + kvm_arm_luo_get_regs(vcpu, &state->regs); + + /* Multiprocessor execution state (uAPI struct kvm_mp_state) */ + kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &state->mp_state); + state->pad = 0; + + /* Exception / SError injection events (uAPI struct kvm_vcpu_events) */ + __kvm_arm_vcpu_get_events(vcpu, &state->events); + + /* CPU target and feature configuration (uAPI struct kvm_vcpu_init) */ + state->init.target = KVM_ARM_TARGET_GENERIC_V8; + bitmap_to_arr32(state->init.features, vcpu->kvm->arch.vcpu_features, + KVM_VCPU_MAX_FEATURES); + + /* System registers (uAPI struct kvm_one_reg array) */ + state->num_sysregs = 0; + state->reserved = 0; + for (i = 0; i < num_sysregs; i++) { + u64 val; + + if (kvm_arm_sys_reg_read(vcpu, indices[i], &val) == 0) { + state->sysregs[state->num_sysregs].id = indices[i]; + state->sysregs[state->num_sysregs].addr = val; + state->num_sysregs++; + } + } + kfree(indices); + + KHOSER_STORE_PTR(ser->arch_state, state); + return 0; +} + +int kvm_arch_vcpu_luo_retrieve(struct kvm_vcpu *vcpu, struct kvm_vcpu_ser *ser) +{ + struct kvm_vcpu_arch_ser *state; + int i; + + if (!ser->arch_state.phys) + return 0; + + state = KHOSER_LOAD_PTR(ser->arch_state); + + /* Restore vCPU features */ + bitmap_from_arr32(vcpu->kvm->arch.vcpu_features, state->init.features, + KVM_VCPU_MAX_FEATURES); + set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &vcpu->kvm->arch.flags); + kvm_reset_vcpu(vcpu); + vcpu_set_flag(vcpu, VCPU_INITIALIZED); + + vcpu_reset_hcr(vcpu); + + /* Restore core registers */ + kvm_arm_luo_set_regs(vcpu, &state->regs); + + if (irqchip_in_kernel(vcpu->kvm) && + vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + vgic_v3_reset(vcpu); + } + + /* Restore system registers */ + for (i = 0; i < state->num_sysregs; i++) + kvm_arm_sys_reg_write(vcpu, state->sysregs[i].id, state->sysregs[i].addr); + + /* Restore multiprocessor execution state */ + kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &state->mp_state); + + /* Restore exception / SError injection events */ + __kvm_arm_vcpu_set_events(vcpu, &state->events); + + if (irqchip_in_kernel(vcpu->kvm)) { + struct vgic_irq *irq; + unsigned long flags; + + vcpu->kvm->arch.vgic.enabled = true; + + irq = vgic_get_vcpu_irq(vcpu, timer_irq(vcpu_vtimer(vcpu))); + if (irq) { + raw_spin_lock_irqsave(&irq->irq_lock, flags); + irq->enabled = true; + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } + irq = vgic_get_vcpu_irq(vcpu, timer_irq(vcpu_ptimer(vcpu))); + if (irq) { + raw_spin_lock_irqsave(&irq->irq_lock, flags); + irq->enabled = true; + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(vcpu->kvm, irq); + } + } + + return 0; +} + +void kvm_arch_vcpu_luo_unpreserve(struct kvm_vcpu_ser *ser) +{ + if (ser->arch_state.phys) { + kho_unpreserve_free(phys_to_virt(ser->arch_state.phys)); + ser->arch_state.phys = 0; + } +} + +void kvm_arch_vcpu_luo_finish(struct kvm_vcpu_ser *ser) +{ + if (ser->arch_state.phys) { + kho_restore_free(phys_to_virt(ser->arch_state.phys)); + ser->arch_state.phys = 0; + } +} diff --git a/include/linux/kho/abi/kvm_arm64.h b/include/linux/kho/abi/kvm_arm64.h new file mode 100644 index 000000000000..dc121224a177 --- /dev/null +++ b/include/linux/kho/abi/kvm_arm64.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#ifndef _LINUX_KHO_ABI_KVM_ARM64_H +#define _LINUX_KHO_ABI_KVM_ARM64_H + +#ifdef CONFIG_ARM64 + +#include <linux/build_bug.h> +#include <linux/stddef.h> +#include <linux/types.h> +#include <linux/kho/abi/kvm.h> +#include <uapi/linux/kvm.h> +#include <uapi/asm/kvm.h> + +/** + * DOC: arm64 KVM vCPU Live Update ABI + * + * arm64 KVM uses the ABI defined below for preserving architectural vCPU state + * across a kexec reboot using LUO. + * + * The state is serialized into a packed structure `struct kvm_vcpu_arch_ser` + * which is handed over to the next kernel via KHO. + * + * The core register structure (struct kvm_regs) is a uAPI contract. + * + * This interface is a contract. Any modification to the structure layout + * constitutes a breaking change. Such changes require incrementing the version + * number in the KVM_VCPU_LUO_FH_COMPATIBLE string. + */ + +/** + * struct kvm_vcpu_arch_ser - Preserved arm64 architectural vCPU state in RAM. + * @regs: Core general-purpose and floating-point registers (uAPI struct kvm_regs). + * @mp_state: Multiprocessor execution state (uAPI struct kvm_mp_state). + * @pad: Padding to maintain 64-bit alignment after mp_state. + * @events: Exception and SError injection state (uAPI struct kvm_vcpu_events). + * @init: Target CPU and vCPU feature bitmap (uAPI struct kvm_vcpu_init). + * @num_sysregs: Number of serialized system registers in sysregs array. + * @reserved: Reserved padding for 64-bit alignment. + * @sysregs: Guest architectural system registers (uAPI struct kvm_one_reg array). + */ +struct kvm_vcpu_arch_ser { + struct kvm_regs regs; + struct kvm_mp_state mp_state; + u32 pad; + struct kvm_vcpu_events events; + struct kvm_vcpu_init init; + u32 num_sysregs; + u32 reserved; + struct kvm_one_reg sysregs[]; +} __packed; + +static_assert(offsetof(struct kvm_vcpu_arch_ser, sysregs) % sizeof(u64) == 0, + "sysregs must be 64-bit aligned"); + +#endif /* CONFIG_ARM64 */ + +#endif /* _LINUX_KHO_ABI_KVM_ARM64_H */ -- 2.55.0.1082.g2b9226bbc0-goog
