Define CPU_PRESERVED_TEXT and CPU_PRESERVED_DATA linker macros to place code and data required for CPU preservation into dedicated sections (.text.cpu_preserved and .data.cpu_preserved).
These sections are preserved across live update transitions and mapped executable/read-write as needed. Signed-off-by: Pasha Tatashin <[email protected]> --- include/asm-generic/vmlinux.lds.h | 29 +++++++++++++++++ include/linux/cpu_preserve.h | 47 +++++++++++++++++++++++++++ scripts/mod/modpost.c | 32 +++++++++++++++--- tools/objtool/check.c | 35 +++++++++++++++++++- tools/objtool/include/objtool/check.h | 1 + tools/objtool/include/objtool/elf.h | 2 +- 6 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 include/linux/cpu_preserve.h diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index b2988aa12f66..0dbaac582bcc 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -655,6 +655,34 @@ *(.static_call.text) \ __static_call_text_end = .; +/* + * Page-aligned text and data sections for preserved CPUs. + * This code and data are KHO preserved when CPUs are preserved across + * live update. + */ +#ifdef CONFIG_LIVEUPDATE_CPU +#define CPU_PRESERVED_TEXT \ + . = ALIGN(PAGE_SIZE); \ + __cpu_preserved_text_start = .; \ + *(.text.cpu_preserved .text.cpu_preserved.*) \ + *(.cpu_preserved.text .cpu_preserved.text.*) \ + . = ALIGN(PAGE_SIZE); \ + __cpu_preserved_text_end = .; + +#define CPU_PRESERVED_DATA \ + . = ALIGN(PAGE_SIZE); \ + __cpu_preserved_data_start = .; \ + *(.data.cpu_preserved .data.cpu_preserved.*) \ + *(.cpu_preserved.data .cpu_preserved.data.*) \ + *(.rodata.cpu_preserved .rodata.cpu_preserved.*) \ + *(.bss..data.cpu_preserved .bss..data.cpu_preserved.*) \ + . = ALIGN(PAGE_SIZE); \ + __cpu_preserved_data_end = .; +#else +#define CPU_PRESERVED_TEXT +#define CPU_PRESERVED_DATA +#endif + /* Section used for early init (in .S files) */ #define HEAD_TEXT KEEP(*(.head.text)) @@ -1155,6 +1183,7 @@ INIT_TASK_DATA(inittask) \ NOSAVE_DATA \ PAGE_ALIGNED_DATA(pagealigned) \ + CPU_PRESERVED_DATA \ CACHE_HOT_DATA(cacheline) \ CACHELINE_ALIGNED_DATA(cacheline) \ READ_MOSTLY_DATA(cacheline) \ diff --git a/include/linux/cpu_preserve.h b/include/linux/cpu_preserve.h new file mode 100644 index 000000000000..f653838c383d --- /dev/null +++ b/include/linux/cpu_preserve.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <[email protected]> + * + * Preserved CPU across Live Update + */ +#ifndef _LINUX_CPU_PRESERVE_H +#define _LINUX_CPU_PRESERVE_H + +#include <linux/compiler.h> + +#ifdef CONFIG_LIVEUPDATE_CPU + +/* + * __cpu_preserved_text: Code executed by preserved physical CPUs during live + * update kexec handover in orphan mode. + * + * All code in this section must run without stack protector checks because + * per-CPU canary state may be invalid during handover and __stack_chk_fail() + * resides in regular .text, which gets overwritten during kexec before the + * incoming kernel boots. + * + * Architecture-specific requirements (such as disabling external retpolines + * and return thunks on x86) are supplied via ARCH_CPU_PRESERVED_TEXT. + */ +#ifndef ARCH_CPU_PRESERVED_TEXT +#define ARCH_CPU_PRESERVED_TEXT +#endif + +#define __cpu_preserved_text \ + __section(".text.cpu_preserved") \ + __no_stack_protector \ + ARCH_CPU_PRESERVED_TEXT +#define __cpu_preserved_data __section(".data.cpu_preserved") + +extern char __cpu_preserved_text_start[], __cpu_preserved_text_end[]; +extern char __cpu_preserved_data_start[], __cpu_preserved_data_end[]; + +#else /* !CONFIG_LIVEUPDATE_CPU */ + +#define __cpu_preserved_text +#define __cpu_preserved_data + +#endif /* CONFIG_LIVEUPDATE_CPU */ + +#endif /* _LINUX_CPU_PRESERVE_H */ diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 75374c64b8cc..51631cd2f6c4 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -810,13 +810,27 @@ static void check_section(struct module *mod, struct elf_info *elf, #define ALL_INIT_SECTIONS ".init.*" #define ALL_EXIT_SECTIONS ".exit.*" +#define ALL_CPU_PRESERVED_TEXT_SECTIONS \ + ".text.cpu_preserved", ".text.cpu_preserved.*", \ + ".cpu_preserved.text", ".cpu_preserved.text.*" + +#define ALL_CPU_PRESERVED_DATA_SECTIONS \ + ".data.cpu_preserved", ".data.cpu_preserved.*", \ + ".cpu_preserved.data", ".cpu_preserved.data.*", \ + ".rodata.cpu_preserved", ".rodata.cpu_preserved.*", \ + ".bss..data.cpu_preserved", ".bss..data.cpu_preserved.*" + +#define ALL_CPU_PRESERVED_SECTIONS \ + ALL_CPU_PRESERVED_TEXT_SECTIONS, ALL_CPU_PRESERVED_DATA_SECTIONS + #define DATA_SECTIONS ".data", ".data.rel" #define TEXT_SECTIONS ".text", ".text.*", ".sched.text", \ ".kprobes.text", ".cpuidle.text", ".noinstr.text", \ ".ltext", ".ltext.*" #define OTHER_TEXT_SECTIONS ".ref.text", ".head.text", ".spinlock.text", \ ".fixup", ".entry.text", ".exception.text", \ - ".coldtext", ".softirqentry.text", ".irqentry.text" + ".coldtext", ".softirqentry.text", ".irqentry.text", \ + ALL_CPU_PRESERVED_TEXT_SECTIONS #define ALL_TEXT_SECTIONS ".init.text", ".exit.text", \ TEXT_SECTIONS, OTHER_TEXT_SECTIONS @@ -827,6 +841,7 @@ enum mismatch { ANY_INIT_TO_ANY_EXIT, ANY_EXIT_TO_ANY_INIT, EXTABLE_TO_NON_TEXT, + CPU_PRESERVED_TO_NON_PRESERVED, }; /** @@ -843,13 +858,19 @@ enum mismatch { * @mismatch: Type of mismatch. */ struct sectioncheck { - const char *fromsec[20]; - const char *bad_tosec[20]; - const char *good_tosec[20]; + const char *fromsec[32]; + const char *bad_tosec[32]; + const char *good_tosec[32]; enum mismatch mismatch; }; static const struct sectioncheck sectioncheck[] = { +/* Do not reference non-preserved code/data from cpu_preserved sections */ +{ + .fromsec = { ALL_CPU_PRESERVED_SECTIONS, NULL }, + .good_tosec = { ALL_CPU_PRESERVED_SECTIONS, NULL }, + .mismatch = CPU_PRESERVED_TO_NON_PRESERVED, +}, /* Do not reference init/exit code/data from * normal code and data */ @@ -960,6 +981,9 @@ static const struct sectioncheck *section_mismatch( static int secref_whitelist(const char *fromsec, const char *fromsym, const char *tosec, const char *tosym) { + if (match(fromsec, PATTERNS(ALL_CPU_PRESERVED_SECTIONS))) + return 1; + /* Check for pattern 1 */ if (match(tosec, PATTERNS(ALL_INIT_DATA_SECTIONS)) && match(fromsec, PATTERNS(DATA_SECTIONS)) && diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 464f6c9d9ff0..77389fbfd9ea 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -329,6 +329,8 @@ static void init_insn_state(struct objtool_file *file, struct insn_state *state, if (opts.noinstr && sec) state->noinstr = sec->noinstr; + if (sec) + state->cpu_preserved = sec->cpu_preserved; } static struct cfi_state *cfi_alloc(void) @@ -419,6 +421,14 @@ static int decode_instructions(struct objtool_file *file) u8 prev_len = 0; u8 idx = 0; + if (!strncmp(sec->name, ".text.cpu_preserved", 19) || + !strncmp(sec->name, ".cpu_preserved.text", 19) || + !strncmp(sec->name, ".data.cpu_preserved", 19) || + !strncmp(sec->name, ".cpu_preserved.data", 19) || + !strncmp(sec->name, ".rodata.cpu_preserved", 21) || + !strncmp(sec->name, ".bss..data.cpu_preserved", 24)) + sec->cpu_preserved = true; + if (!is_text_sec(sec)) continue; @@ -3511,6 +3521,17 @@ static int validate_call(struct objtool_file *file, struct instruction *insn, struct insn_state *state) { + if (state->cpu_preserved) { + struct symbol *dest = insn_call_dest(insn); + + if (dest && (dest->sec->idx != SHN_UNDEF || opts.link) && + !dest->sec->cpu_preserved) { + WARN_INSN(insn, "call to %s() leaves .text.cpu_preserved section", + call_dest_name(insn)); + return 1; + } + } + if (state->noinstr && state->instr <= 0 && !noinstr_call_dest(file, insn, insn_call_dest(insn))) { WARN_INSN(insn, "call to %s() leaves .noinstr.text section", call_dest_name(insn)); @@ -4164,7 +4185,13 @@ static int validate_retpoline(struct objtool_file *file) if (insn->retpoline_safe) continue; - if (insn->sec->init) + /* + * Preserved CPU text (.text.cpu_preserved) executes across + * kexec when the outgoing kernel's retpoline/rethunk targets + * are no longer mapped. + */ + if (insn->sec->init || + !strcmp(insn->sec->name, ".text.cpu_preserved")) continue; if (insn->type == INSN_RETURN) { @@ -4440,6 +4467,12 @@ static int validate_noinstr_sections(struct objtool_file *file) warnings += validate_unwind_hints(file, sec); } + sec = find_section_by_name(file->elf, ".text.cpu_preserved"); + if (sec) { + warnings += validate_section(file, sec); + warnings += validate_unwind_hints(file, sec); + } + return warnings; } diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index 063f5985fecd..18318f2faf11 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -16,6 +16,7 @@ struct insn_state { bool uaccess; bool df; bool noinstr; + bool cpu_preserved; s8 instr; }; diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index a82517a76a0f..bac9e5f4921f 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -58,7 +58,7 @@ struct section { Elf_Data *data; const char *name; int idx; - bool _changed, text, rodata, noinstr, init, truncate; + bool _changed, text, rodata, noinstr, init, cpu_preserved, truncate; struct reloc *relocs; unsigned long nr_alloc_relocs; struct section *twin; -- 2.55.0.1082.g2b9226bbc0-goog
