On HAVE_RCU_TRAMPOLINE_READERS kernels Tasks RCU keeps a BPF trampoline image allocated only while a task using it is a Tasks Trace RCU reader or is executing text that rcu_tasks_trampoline_text() recognises. The image itself is such text, but the C glue and the programs it calls are not, and only sleepable programs take rcu_read_lock_trace() today.
Have the x86-64 JIT open-code rcu_read_lock_trace() and rcu_read_unlock_trace() in the trampoline, as ftrace_64.S does for ftrace_caller: one reader from just after the frame is set up to just before the original function is called, covering __bpf_tramp_enter() and the fentry and fmod_ret programs, and a second one from just after the original function returns to just before the final register restore, covering the fexit programs and __bpf_tramp_exit(). The original function itself runs outside both, since it may run for a long time and the image is pinned by im->pcref across it. Trampolines that do not call the original function get a single reader around all their programs. The second reader is entered before ip_after_call, so the ip_after_call -> ip_epilogue jump that bpf_tramp_image_put() patches in is inside it, and the fmod_ret early-exit branch lands after that point still holding the first reader, so exactly one is held on every path. The sequence uses r10 and r11, which are scratch at each emission point, and references current_task and rcu_tasks_trace_srcu_struct by absolute sign-extended address, the form the JIT already relies on for this_cpu_off. Sleepable programs' own rcu_read_lock_trace() simply nests. Nothing is emitted on other configurations. Suggested-by: Alexei Starovoitov <[email protected]> Assisted-by: LLM Signed-off-by: Josef Bacik <[email protected]> --- arch/x86/net/bpf_jit_comp.c | 113 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2853e87797a7..c991f7ceacdf 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -14,6 +14,7 @@ #include <linux/memory.h> #include <linux/sort.h> #include <linux/execmem.h> +#include <linux/rcupdate_trace.h> #include <asm/extable.h> #include <asm/ftrace.h> #include <asm/set_memory.h> @@ -722,6 +723,97 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip) *pprog = prog; } +/* + * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the + * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent + * macros in arch/x86/kernel/ftrace_64.S. The image is not relocated, so + * current_task and rcu_tasks_trace_srcu_struct are referenced by absolute + * (sign-extended 32-bit) address, the form the JIT already relies on for + * this_cpu_off. Uses r10 and r11, which are scratch at every emission + * point, and clobbers flags. + * + * lock: unlock: + * mov r11, gs:[current_task] mov r11, gs:[current_task] + * mov r10d, [r11+nesting] mov r10d, [r11+nesting] + * inc dword ptr [r11+nesting] sub r10d, 1 + * test r10d, r10d jnz 2f + * jnz 1f mov r10, [r11+scp] + * mov r10, [&srcu.srcu_ctrp] mov dword ptr [r11+nesting], 0 + * inc qword ptr gs:[r10+locks] (smp_mb) + * mov [r11+scp], r10 inc qword ptr gs:[r10+unlocks] + * (smp_mb) jmp 3f + * 1: 2: mov [r11+nesting], r10d + * 3: + */ +static void emit_trace_rcu_reader(u8 **pprog, bool lock) +{ +#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS + const u32 nesting = offsetof(struct task_struct, trc_reader_nesting); + const u32 scp = offsetof(struct task_struct, trc_reader_scp); + const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB); + u8 *prog = *pprog; + + BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)); + BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_locks) != 0); + BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_unlocks) != 8); + + /* mov r11, gs:[abs32 current_task] */ + EMIT2(0x65, 0x4C); + EMIT3(0x8B, 0x1C, 0x25); + EMIT((u32)(unsigned long)¤t_task, 4); + /* mov r10d, dword ptr [r11 + nesting] */ + EMIT3(0x45, 0x8B, 0x93); + EMIT(nesting, 4); + + if (lock) { + /* inc dword ptr [r11 + nesting] */ + EMIT3(0x41, 0xFF, 0x83); + EMIT(nesting, 4); + /* test r10d, r10d */ + EMIT3(0x45, 0x85, 0xD2); + /* jnz 1f */ + EMIT2(X86_JNE, 8 + 4 + 7 + (mb ? 6 : 0)); + /* mov r10, qword ptr [abs32 &rcu_tasks_trace_srcu_struct.srcu_ctrp] */ + EMIT4(0x4C, 0x8B, 0x14, 0x25); + EMIT((u32)(unsigned long)&rcu_tasks_trace_srcu_struct.srcu_ctrp, 4); + /* inc qword ptr gs:[r10] */ + EMIT4(0x65, 0x49, 0xFF, 0x02); + /* mov qword ptr [r11 + scp], r10 */ + EMIT3(0x4D, 0x89, 0x93); + EMIT(scp, 4); + /* smp_mb(): lock add dword ptr [rsp - 4], 0 */ + if (mb) + EMIT2_off32(0xF0, 0x83, 0x00FC2444); + /* 1: */ + } else { + /* sub r10d, 1 */ + EMIT4(0x41, 0x83, 0xEA, 0x01); + /* jnz 2f */ + EMIT2(X86_JNE, 7 + 11 + (mb ? 6 : 0) + 5 + 2); + /* mov r10, qword ptr [r11 + scp] */ + EMIT3(0x4D, 0x8B, 0x93); + EMIT(scp, 4); + /* mov dword ptr [r11 + nesting], 0 */ + EMIT3(0x41, 0xC7, 0x83); + EMIT(nesting, 4); + EMIT(0, 4); + if (mb) + EMIT2_off32(0xF0, 0x83, 0x00FC2444); + /* inc qword ptr gs:[r10 + 8] */ + EMIT4(0x65, 0x49, 0xFF, 0x42); + EMIT1(0x08); + /* jmp 3f */ + EMIT2(0xEB, 7); + /* 2: mov dword ptr [r11 + nesting], r10d */ + EMIT3(0x45, 0x89, 0x93); + EMIT(nesting, 4); + /* 3: */ + } + + *pprog = prog; +#endif +} + static void emit_return(u8 **pprog, u8 *ip) { u8 *prog = *pprog; @@ -3610,6 +3702,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* mov QWORD PTR [rbp - rbx_off], rbx */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off); + /* + * Tasks RCU keeps this image alive only while we are a Tasks Trace + * reader; the instructions before this point (and after the final + * unlock) are covered by the irq-exit IP check. One reader spans + * __bpf_tramp_enter() and the fentry/fmod_ret progs, a second one + * the fexit progs and __bpf_tramp_exit(); the original function runs + * outside both, with the image pinned by im->pcref instead. + */ + emit_trace_rcu_reader(&prog, true); + func_meta = nr_regs; /* Store number of argument registers of the traced function */ emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); @@ -3660,6 +3762,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { + emit_trace_rcu_reader(&prog, false); restore_regs(m, &prog, regs_off); save_args(m, &prog, arg_stack_off, true, flags, 0); @@ -3682,6 +3785,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); + /* + * Second reader. Taken before ip_after_call so that the + * ip_after_call -> ip_epilogue jump patched in at teardown is + * inside it too; the fmod_ret early exit jumps past this still + * holding the first reader, so either way exactly one is held. + */ + emit_trace_rcu_reader(&prog, true); im->ip_after_call = image + (prog - (u8 *)rw_image); emit_nops(&prog, X86_PATCH_SIZE); } @@ -3737,6 +3847,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size); } + /* Remaining instructions are covered by the irq-exit IP check. */ + emit_trace_rcu_reader(&prog, false); + /* restore return value of orig_call or fentry prog back into RAX */ if (save_ret) emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8); -- 2.55.0
