From: Abhishek Dubey <[email protected]> To support tailcalls in subprogs, tail_call_cnt needs to be on the BPF trampoline stack frame. In a regular BPF program or subprog stack frame, the position of tail_call_cnt is after the NVR save area (BPF_PPC_STACK_SAVE). To avoid complex logic in deducing offset for tail_call_cnt, it has to be kept at the same offset on the trampoline frame as well. But doing that wastes nearly all of BPF_PPC_STACK_SAVE bytes on the BPF trampoline stack frame as the NVR save area is not the same for BPF trampoline and regular BPF programs. Address this by moving tail_call_cnt to the bottom of the frame.
This change avoids the need to account for BPF_PPC_STACK_SAVE bytes in the BPF trampoline stack frame when support for tailcalls in BPF subprogs is added later. Also, this change makes offset calculation of tail_call_cnt field simpler all across. Signed-off-by: Abhishek Dubey <[email protected]> --- arch/powerpc/net/bpf_jit.h | 1 + arch/powerpc/net/bpf_jit_comp.c | 6 +++--- arch/powerpc/net/bpf_jit_comp64.c | 31 ++++++++++++++++++++----------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 8334cd667bba..9f6ec00bd02e 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -24,6 +24,7 @@ #define SZL sizeof(unsigned long) #define BPF_INSN_SAFETY 64 +#define BPF_PPC_TAILCALL 8 #define PLANT_INSTR(d, idx, instr) \ do { if (d) { (d)[idx] = instr; } idx++; } while (0) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 5e976730b2f5..de1adc0b1157 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -604,8 +604,8 @@ static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_contex int func_frame_offset, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { - /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 7 * 8; + /* See Generated stack layout */ + int tailcallcnt_offset = BPF_PPC_TAILCALL; EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset)); @@ -620,7 +620,7 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont { if (IS_ENABLED(CONFIG_PPC64)) { /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 7 * 8; + int tailcallcnt_offset = BPF_PPC_TAILCALL; EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 1fe37128c876..296e9ea14f2e 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -20,13 +20,15 @@ #include "bpf_jit.h" /* - * Stack layout: + * Stack layout with frame: + * Layout when setting up our own stack frame. + * Note: r1 at bottom, component offsets positive wrt r1. * Ensure the top half (upto local_tmp_var) stays consistent * with our redzone usage. * * [ prev sp ] <------------- - * [ nv gpr save area ] 6*8 | * [ tail_call_cnt ] 8 | + * [ nv gpr save area ] 6*8 | * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 | @@ -36,10 +38,12 @@ /* for gpr non volatile registers BPG_REG_6 to 10 */ #define BPF_PPC_STACK_SAVE (6*8) /* for bpf JIT code internal usage */ -#define BPF_PPC_STACK_LOCALS 32 +#define BPF_PPC_STACK_LOCALS 24 /* stack frame excluding BPF stack, ensure this is quadword aligned */ #define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \ - BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE) + BPF_PPC_STACK_LOCALS + \ + BPF_PPC_STACK_SAVE + \ + BPF_PPC_TAILCALL) /* BPF register usage */ #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) @@ -87,27 +91,32 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) } /* + * Stack layout with redzone: * When not setting up our own stackframe, the redzone (288 bytes) usage is: + * Note: r1 from prev frame. Component offset negative wrt r1. * * [ prev sp ] <------------- * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- - * [ nv gpr save area ] 6*8 * [ tail_call_cnt ] 8 + * [ nv gpr save area ] 6*8 * [ local_tmp_var ] 24 * [ unused red zone ] 224 */ static int bpf_jit_stack_local(struct codegen_context *ctx) { - if (bpf_has_stack_frame(ctx)) + if (bpf_has_stack_frame(ctx)) { + /* Stack layout with frame */ return STACK_FRAME_MIN_SIZE + ctx->stack_size; - else - return -(BPF_PPC_STACK_SAVE + 32); + } else { + /* Stack layout with redzone */ + return -(BPF_PPC_TAILCALL + BPF_PPC_STACK_SAVE + BPF_PPC_STACK_LOCALS); + } } static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) { - return bpf_jit_stack_local(ctx) + 24; + return bpf_jit_stack_local(ctx) + BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE; } static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) @@ -115,7 +124,7 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) if (reg >= BPF_PPC_NVR_MIN && reg < 32) return (bpf_has_stack_frame(ctx) ? (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) - - (8 * (32 - reg)); + - (8 * (32 - reg)) - BPF_PPC_TAILCALL; pr_err("BPF JIT is asking about unknown registers"); BUG(); @@ -145,7 +154,7 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) if (ctx->seen & SEEN_TAILCALL) { EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); /* this goes in the redzone */ - EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_STACK_SAVE + 8))); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); } else { EMIT(PPC_RAW_NOP()); EMIT(PPC_RAW_NOP()); -- 2.48.1
