Convert every possible CPU's decrypted section after per-CPU setup and before the boot CPU registers its buffers. This replaces KVM's object loop and shares VMware steal-time storage without a driver conversion path or readiness state.
UP KVM registers inside setup_arch(), so convert before guest_late_init() there and move VMware's UP registration to that hook. Stop boot on a conversion failure: inconsistent page state must not be published to the hypervisor. Host SME keeps its existing mappings. Skip Hyper-V vTOM per-CPU storage: its visibility callbacks require later Hyper-V initialization. Suggested-by: Kiryl Shutsemau <[email protected]> Link: https://lore.kernel.org/r/aqqGUAX65s4LdJkr@thinkstation Link: https://lore.kernel.org/r/aqvxQoIoYhJTZpAC@thinkstation Signed-off-by: Zack Rusin <[email protected]> --- arch/x86/hyperv/ivm.c | 4 ++++ arch/x86/include/asm/mem_encrypt.h | 2 ++ arch/x86/include/asm/x86_init.h | 2 ++ arch/x86/kernel/cpu/vmware.c | 2 +- arch/x86/kernel/kvm.c | 35 ----------------------------------- arch/x86/kernel/setup.c | 2 ++ arch/x86/kernel/setup_percpu.c | 2 ++ arch/x86/mm/mem_encrypt.c | 22 ++++++++++++++++++++++ 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c index 2ce4dfe53472..4a5c735c9c52 100644 --- a/arch/x86/hyperv/ivm.c +++ b/arch/x86/hyperv/ivm.c @@ -887,6 +887,10 @@ void __init hv_vtom_init(void) cc_set_mask(ms_hyperv.shared_gpa_boundary); physical_mask &= ms_hyperv.shared_gpa_boundary - 1; + /* vTOM has no early per-CPU consumers and needs the Hyper-V setup. */ + x86_init.paging.skip_percpu_decryption = true; + x86_init.paging.early_decrypt_page = NULL; + x86_platform.hyper.is_private_mmio = hv_is_private_mmio; x86_platform.guest.enc_cache_flush_required = hv_vtom_cache_flush_required; x86_platform.guest.enc_tlb_flush_required = hv_vtom_tlb_flush_required; diff --git a/arch/x86/include/asm/mem_encrypt.h b/arch/x86/include/asm/mem_encrypt.h index 4d81f693b1d3..05cf395407ef 100644 --- a/arch/x86/include/asm/mem_encrypt.h +++ b/arch/x86/include/asm/mem_encrypt.h @@ -21,11 +21,13 @@ struct boot_params; #ifdef CONFIG_X86_MEM_ENCRYPT void __init mem_encrypt_init(void); void __init mem_encrypt_setup_arch(void); +void __init mem_encrypt_init_percpu(void); int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size); void __init early_set_page_decrypted(unsigned long addr, unsigned long alias); #else static inline void mem_encrypt_init(void) { } static inline void __init mem_encrypt_setup_arch(void) { } +static inline void __init mem_encrypt_init_percpu(void) { } static inline int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) { return 0; } #endif diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index e4131402c783..8d1597372eb6 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -76,10 +76,12 @@ struct x86_init_oem { * Callback must call paging_init(). Called once after the * direct mapping for phys memory is available. * @early_decrypt_page: Share a direct-mapped page and its optional image alias + * @skip_percpu_decryption: Platform does not use early shared per-CPU data */ struct x86_init_paging { void (*pagetable_init)(void); int (*early_decrypt_page)(unsigned long addr, unsigned long alias); + bool skip_percpu_decryption; }; /** diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c index 34b73573b108..b477cc027b18 100644 --- a/arch/x86/kernel/cpu/vmware.c +++ b/arch/x86/kernel/cpu/vmware.c @@ -366,7 +366,7 @@ static void __init vmware_paravirt_ops_setup(void) vmware_cpu_down_prepare) < 0) pr_err("vmware_guest: Failed to install cpu hotplug callbacks\n"); #else - vmware_guest_cpu_init(); + x86_init.hyper.guest_late_init = vmware_guest_cpu_init; #endif } } diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6b0a5861ccb8..acb3b7b18ebe 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -429,34 +429,6 @@ static u64 kvm_steal_clock(int cpu) return steal; } -static inline __init void __set_percpu_decrypted(void *ptr, unsigned long size) -{ - early_set_memory_decrypted((unsigned long) ptr, size); -} - -/* - * Iterate through all possible CPUs and map the memory region pointed - * by apf_reason, steal_time and kvm_apic_eoi as decrypted at once. - * - * Note: we iterate through all possible CPUs to ensure that CPUs - * hotplugged will have their per-cpu variable already mapped as - * decrypted. - */ -static void __init sev_map_percpu_data(void) -{ - int cpu; - - if (cc_vendor != CC_VENDOR_AMD || - !cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) - return; - - for_each_possible_cpu(cpu) { - __set_percpu_decrypted(&per_cpu(apf_reason, cpu), sizeof(apf_reason)); - __set_percpu_decrypted(&per_cpu(steal_time, cpu), sizeof(steal_time)); - __set_percpu_decrypted(&per_cpu(kvm_apic_eoi, cpu), sizeof(kvm_apic_eoi)); - } -} - static void kvm_guest_cpu_offline(bool shutdown) { kvm_disable_steal_time(); @@ -709,12 +681,6 @@ arch_initcall(kvm_alloc_cpumask); static void __init kvm_smp_prepare_boot_cpu(void) { - /* - * Map the per-cpu variables as decrypted before kvm_guest_cpu_init() - * shares the guest physical address with the hypervisor. - */ - sev_map_percpu_data(); - kvm_guest_cpu_init(); native_smp_prepare_boot_cpu(); kvm_spinlock_init(); @@ -868,7 +834,6 @@ static void __init kvm_guest_init(void) kvm_cpu_online, kvm_cpu_down_prepare) < 0) pr_err("failed to install cpu hotplug callbacks\n"); #else - sev_map_percpu_data(); kvm_guest_cpu_init(); #endif diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index cda6adb9f69c..8eebd85e59af 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1251,6 +1251,8 @@ void __init setup_arch(char **cmdline_p) io_apic_init_mappings(); + if (!IS_ENABLED(CONFIG_SMP)) + mem_encrypt_init_percpu(); x86_init.hyper.guest_late_init(); e820__reserve_resources(); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index c83c61e0b20a..8526ad37a81b 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -5,6 +5,7 @@ #include <linux/export.h> #include <linux/init.h> #include <linux/memblock.h> +#include <linux/mem_encrypt.h> #include <linux/percpu.h> #include <linux/kexec.h> #include <linux/crash_dump.h> @@ -234,4 +235,5 @@ void __init setup_per_cpu_areas(void) * this call? */ sync_initial_page_table(); + mem_encrypt_init_percpu(); } diff --git a/arch/x86/mm/mem_encrypt.c b/arch/x86/mm/mem_encrypt.c index c3e239a47b66..d3224607170d 100644 --- a/arch/x86/mm/mem_encrypt.c +++ b/arch/x86/mm/mem_encrypt.c @@ -13,6 +13,7 @@ #include <linux/cc_platform.h> #include <linux/mem_encrypt.h> #include <linux/pgalloc.h> +#include <linux/percpu.h> #include <linux/virtio_anchor.h> #include <linux/iommu-dma.h> @@ -24,6 +25,8 @@ #include "mm_internal.h" +extern char __percpu __start_percpu_decrypted[], __end_percpu_decrypted[]; + static pte_t * __init early_lookup_pte(unsigned long addr) { unsigned long pfn, step; @@ -134,6 +137,25 @@ int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size) return 0; } +void __init mem_encrypt_init_percpu(void) +{ + unsigned long size = __end_percpu_decrypted - __start_percpu_decrypted; + int cpu, ret; + + if (!cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT) || + x86_init.paging.skip_percpu_decryption) + return; + + for_each_possible_cpu(cpu) { + unsigned long addr = (unsigned long) + per_cpu_ptr(__start_percpu_decrypted, cpu); + + ret = early_set_memory_decrypted(addr, size); + if (ret) + panic("Cannot share CPU %d per-CPU data (err=%d)", cpu, ret); + } +} + /* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */ bool force_dma_unencrypted(struct device *dev) { -- 2.53.0
