Add arch/x86/kernel/preserve_cpu.S stack switch routine, preserved linker sections in vmlinux.lds.S, and SMP stop exclusion for preserved cores.
Signed-off-by: Pasha Tatashin <[email protected]> --- arch/x86/include/asm/cpu_preserve.h | 31 +++++++++++++++ arch/x86/kernel/preserve_cpu.S | 62 +++++++++++++++++++++++++++++ arch/x86/kernel/vmlinux.lds.S | 2 + 3 files changed, 95 insertions(+) create mode 100644 arch/x86/include/asm/cpu_preserve.h create mode 100644 arch/x86/kernel/preserve_cpu.S diff --git a/arch/x86/include/asm/cpu_preserve.h b/arch/x86/include/asm/cpu_preserve.h new file mode 100644 index 000000000000..969903eb70cb --- /dev/null +++ b/arch/x86/include/asm/cpu_preserve.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#ifndef __ASM_X86_CPU_PRESERVE_H +#define __ASM_X86_CPU_PRESERVE_H + +#include <asm/page_types.h> + +#define ARCH_CPU_PRESERVED_STACK_ORDER THREAD_SIZE_ORDER + +#ifdef CONFIG_CC_IS_GCC +#define ARCH_CPU_PRESERVED_TEXT \ + __attribute__((indirect_branch("keep"), function_return("keep"))) +#else +#define ARCH_CPU_PRESERVED_TEXT +#endif + +#ifdef CONFIG_LIVEUPDATE_CPU +void arch_cpu_preserved_load_desc(void); +bool arch_cpu_preserved_is_active(void); +void x86_preserved_iret_stub(void); +void x86_preserved_iret_err_stub(void); +void x86_preserved_apic_eoi_stub(void); +#else +static inline void arch_cpu_preserved_load_desc(void) {} +static inline bool arch_cpu_preserved_is_active(void) { return false; } +#endif + +#endif /* __ASM_X86_CPU_PRESERVE_H */ diff --git a/arch/x86/kernel/preserve_cpu.S b/arch/x86/kernel/preserve_cpu.S new file mode 100644 index 000000000000..8acb98d46f90 --- /dev/null +++ b/arch/x86/kernel/preserve_cpu.S @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + */ +#include <linux/linkage.h> +#include <asm/nospec-branch.h> +#include <asm/unwind_hints.h> + + .section .text.cpu_preserved, "ax" + +SYM_CODE_START(x86_preserved_iret_err_stub) + UNWIND_HINT_IRET_REGS + ENDBR + addq $8, %rsp + iretq +SYM_CODE_END(x86_preserved_iret_err_stub) + +SYM_CODE_START(x86_preserved_iret_stub) + UNWIND_HINT_IRET_REGS + ENDBR + iretq +SYM_CODE_END(x86_preserved_iret_stub) + +SYM_CODE_START(x86_preserved_apic_eoi_stub) + UNWIND_HINT_IRET_REGS + ENDBR + pushq %rax + pushq %rcx + pushq %rdx + /* x2APIC mode: MSR 0x80b (APIC_EOI) = 0 */ + movl $0x80b, %ecx + xorl %eax, %eax + xorl %edx, %edx + wrmsr + popq %rdx + popq %rcx + popq %rax + iretq +SYM_CODE_END(x86_preserved_apic_eoi_stub) + +/* + * arch_cpu_preserved_call_on_stack - Switch stack and call function + * @cpu: CPU ID (%rdi) + * @stack: Target stack top (%rsi) + * @fn: Function pointer to invoke (%rdx) + */ +SYM_CODE_START(arch_cpu_preserved_call_on_stack) + pushq %rbp + movq %rsp, %rbp + pushq %r12 + + movq %rsp, %r12 /* save old stack pointer */ + movq %rsi, %rsp /* switch to target stack */ + + call *%rdx /* call fn(cpu) */ + + movq %r12, %rsp /* restore old stack pointer */ + popq %r12 + leave + ret +SYM_CODE_END(arch_cpu_preserved_call_on_stack) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 2438b89a4620..9b8b70057bcf 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -155,6 +155,7 @@ SECTIONS #endif ALIGN_ENTRY_TEXT_END + CPU_PRESERVED_TEXT TEXT_TEXT SCHED_TEXT LOCK_TEXT @@ -194,6 +195,7 @@ SECTIONS #endif PAGE_ALIGNED_DATA(PAGE_SIZE) + CPU_PRESERVED_DATA CACHE_HOT_DATA(L1_CACHE_BYTES) -- 2.55.0.1082.g2b9226bbc0-goog
