In the non-trampoline SYSCALL64 path, we use a percpu variable to temporarily store the user RSP value. Instead of a separate variable, use the otherwise unused sp2 slot in the TSS. This will improve cache locality, as the sp1 slot is already used in the same code to find the kernel stack. It will also simplify a future change to make the non-trampoline path work in PTI mode.
Signed-off-by: Andy Lutomirski <[email protected]> --- arch/x86/include/asm/processor.h | 5 +++++ arch/x86/include/asm/thread_info.h | 1 + arch/x86/kernel/asm-offsets_64.c | 1 + arch/x86/kernel/process_64.c | 2 -- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index cfd29ee8c3da..2ef4c39ded45 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -308,7 +308,12 @@ struct x86_hw_tss { */ u64 sp1; + /* + * sp2 is scratch space used by the SYSCALL64 handler. Linux does + * not use rung 2, so sp2 is not otherwise needed. + */ u64 sp2; + u64 reserved2; u64 ist[7]; u32 reserved3; diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 2ff2a30a264f..9a2f84233e39 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -209,6 +209,7 @@ static inline int arch_within_stack_frames(const void * const stack, #ifdef CONFIG_X86_64 # define cpu_current_top_of_stack (cpu_tss_rw + TSS_sp1) +# define rsp_scratch (cpu_tss_rw + TSS_sp2) #endif #endif diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index b2dcd161f514..621bf6b5a63b 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -67,6 +67,7 @@ int main(void) OFFSET(TSS_ist, tss_struct, x86_tss.ist); OFFSET(TSS_sp0, tss_struct, x86_tss.sp0); OFFSET(TSS_sp1, tss_struct, x86_tss.sp1); + OFFSET(TSS_sp2, tss_struct, x86_tss.sp2); BLANK(); #ifdef CONFIG_STACKPROTECTOR diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 12bb445fb98d..3ed5fed181cc 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -59,8 +59,6 @@ #include <asm/unistd_32_ia32.h> #endif -__visible DEFINE_PER_CPU(unsigned long, rsp_scratch); - /* Prints also some state that isn't saved in the pt_regs */ void __show_regs(struct pt_regs *regs, int all) { -- 2.17.1
