From: He Kuang <heku...@huawei.com> This patch generates prologue for a BPF program which fetch arguments for it. With this patch, the program can have arguments as follow:
SEC("lock_page=__lock_page page->flags") int lock_page(struct pt_regs *ctx, int err, unsigned long flags) { return 1; } This patch passes at most 3 arguments from r3, r4 and r5. r1 is still the ctx pointer. r2 is used to indicate the successfulness of dereferencing. This patch uses r6 to hold ctx (struct pt_regs) and r7 to hold stack pointer for result. Result of each arguments first store on stack: low address BPF_REG_FP - 24 ARG3 BPF_REG_FP - 16 ARG2 BPF_REG_FP - 8 ARG1 BPF_REG_FP high address Then loaded into r3, r4 and r5. The output prologue for offn(...off2(off1(reg)))) should be: r6 <- r1 // save ctx into a callee saved register r7 <- fp r7 <- r7 - stack_offset // pointer to result slot /* load r3 with the offset in pt_regs of 'reg' */ (r7) <- r3 // make slot valid r3 <- r3 + off1 // prepare to read unsafe pointer r2 <- 8 r1 <- r7 // result put onto stack call probe_read // read unsafe pointer jnei r0, 0, err // error checking r3 <- (r7) // read result r3 <- r3 + off2 // prepare to read unsafe pointer r2 <- 8 r1 <- r7 call probe_read jnei r0, 0, err ... /* load r2, r3, r4 from stack */ goto success err: r2 <- 1 /* load r3, r4, r5 with 0 */ goto usercode success: r2 <- 0 usercode: r1 <- r6 // restore ctx // original user code If all of arguments reside in register (dereferencing is not required), gen_prologue_fastpath() will be used to create fast prologue: r3 <- (r1 + offset of reg1) r4 <- (r1 + offset of reg2) r5 <- (r1 + offset of reg3) r2 <- 0 P.S. eBPF calling convention is defined as: * r0 - return value from in-kernel function, and exit value for eBPF program * r1 - r5 - arguments from eBPF program to in-kernel function * r6 - r9 - callee saved registers that in-kernel function will preserve * r10 - read-only frame pointer to access stack Signed-off-by: He Kuang <heku...@huawei.com> Signed-off-by: Wang Nan <wangn...@huawei.com> Cc: Alexei Starovoitov <a...@plumgrid.com> Cc: Brendan Gregg <brendan.d.gr...@gmail.com> Cc: Daniel Borkmann <dan...@iogearbox.net> Cc: David Ahern <dsah...@gmail.com> Cc: He Kuang <heku...@huawei.com> Cc: Jiri Olsa <jo...@kernel.org> Cc: Kaixu Xia <xiaka...@huawei.com> Cc: Masami Hiramatsu <masami.hiramatsu...@hitachi.com> Cc: Namhyung Kim <namhy...@kernel.org> Cc: Paul Mackerras <pau...@samba.org> Cc: Peter Zijlstra <a.p.zijls...@chello.nl> Cc: Zefan Li <lize...@huawei.com> Cc: pi3or...@163.com Cc: Arnaldo Carvalho de Melo <a...@redhat.com> Link: http://lkml.kernel.org/n/ebpf-6yw9eg0ej3l4jnqhinngk...@git.kernel.org --- tools/perf/util/Build | 1 + tools/perf/util/bpf-prologue.c | 443 +++++++++++++++++++++++++++++++++++++++++ tools/perf/util/bpf-prologue.h | 34 ++++ 3 files changed, 478 insertions(+) create mode 100644 tools/perf/util/bpf-prologue.c create mode 100644 tools/perf/util/bpf-prologue.h diff --git a/tools/perf/util/Build b/tools/perf/util/Build index b808151..579cf84 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -87,6 +87,7 @@ libperf-y += parse-branch-options.o libperf-y += parse-regs-options.o libperf-$(CONFIG_LIBBPF) += bpf-loader.o +libperf-$(CONFIG_BPF_PROLOGUE) += bpf-prologue.o libperf-$(CONFIG_LIBELF) += symbol-elf.o libperf-$(CONFIG_LIBELF) += probe-file.o libperf-$(CONFIG_LIBELF) += probe-event.o diff --git a/tools/perf/util/bpf-prologue.c b/tools/perf/util/bpf-prologue.c new file mode 100644 index 0000000..e4adb18 --- /dev/null +++ b/tools/perf/util/bpf-prologue.c @@ -0,0 +1,443 @@ +/* + * bpf-prologue.c + * + * Copyright (C) 2015 He Kuang <heku...@huawei.com> + * Copyright (C) 2015 Wang Nan <wangn...@huawei.com> + * Copyright (C) 2015 Huawei Inc. + */ + +#include <bpf/libbpf.h> +#include "perf.h" +#include "debug.h" +#include "bpf-prologue.h" +#include "probe-finder.h" +#include <dwarf-regs.h> +#include <linux/filter.h> + +#define BPF_REG_SIZE 8 + +#define JMP_TO_ERROR_CODE -1 +#define JMP_TO_SUCCESS_CODE -2 +#define JMP_TO_USER_CODE -3 + +struct bpf_insn_pos { + struct bpf_insn *begin; + struct bpf_insn *end; + struct bpf_insn *pos; +}; + +static inline int +pos_get_cnt(struct bpf_insn_pos *pos) +{ + return pos->pos - pos->begin; +} + +static int +append_insn(struct bpf_insn new_insn, struct bpf_insn_pos *pos) +{ + if (!pos->pos) + return -ERANGE; + + if (pos->pos + 1 >= pos->end) { + pr_err("bpf prologue: prologue too long\n"); + pos->pos = NULL; + return -ERANGE; + } + + *(pos->pos)++ = new_insn; + return 0; +} + +static int +check_pos(struct bpf_insn_pos *pos) +{ + if (!pos->pos || pos->pos >= pos->end) + return -ERANGE; + return 0; +} + +/* Give it a shorter name */ +#define ins(i, p) append_insn((i), (p)) + +/* + * Give a register name (in 'reg'), generate instruction to + * load register into an eBPF register rd: + * 'ldd target_reg, offset(ctx_reg)', where: + * ctx_reg is pre initialized to pointer of 'struct pt_regs'. + */ +static int +gen_ldx_reg_from_ctx(struct bpf_insn_pos *pos, int ctx_reg, + const char *reg, int target_reg) +{ + int offset = regs_query_register_offset(reg); + + if (offset < 0) { + pr_err("bpf: prologue: failed to get register %s\n", + reg); + return -1; + } + ins(BPF_LDX_MEM(BPF_DW, target_reg, ctx_reg, offset), pos); + + if (check_pos(pos)) + return -ERANGE; + return 0; +} + +/* + * Generate a BPF_FUNC_probe_read function call. + * + * src_base_addr_reg is a register holding base address, + * dst_addr_reg is a register holding dest address (on stack), + * result is: + * + * *[dst_addr_reg] = *([src_base_addr_reg] + offset) + * + * Arguments of BPF_FUNC_probe_read: + * ARG1: ptr to stack (dest) + * ARG2: size (8) + * ARG3: unsafe ptr (src) + */ +static int +gen_read_mem(struct bpf_insn_pos *pos, + int src_base_addr_reg, + int dst_addr_reg, + long offset) +{ + /* mov arg3, src_base_addr_reg */ + if (src_base_addr_reg != BPF_REG_ARG3) + ins(BPF_MOV64_REG(BPF_REG_ARG3, src_base_addr_reg), pos); + /* add arg3, #offset */ + if (offset) + ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_ARG3, offset), pos); + + /* mov arg2, #reg_size */ + ins(BPF_ALU64_IMM(BPF_MOV, BPF_REG_ARG2, BPF_REG_SIZE), pos); + + /* mov arg1, dst_addr_reg */ + if (dst_addr_reg != BPF_REG_ARG1) + ins(BPF_MOV64_REG(BPF_REG_ARG1, dst_addr_reg), pos); + + /* Call probe_read */ + ins(BPF_EMIT_CALL(BPF_FUNC_probe_read), pos); + /* + * Error processing: if read fail, goto error code, + * will be relocated. Target should be the start of + * error processing code. + */ + ins(BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, JMP_TO_ERROR_CODE), + pos); + + if (check_pos(pos)) + return -ERANGE; + return 0; +} + +/* + * Each arg should be bare register. Fetch and save them into argument + * registers (r3 - r5). + * + * BPF_REG_1 should have been initialized with pointer to + * 'struct pt_regs'. + */ +static int +gen_prologue_fastpath(struct bpf_insn_pos *pos, + struct probe_trace_arg *args, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) + if (gen_ldx_reg_from_ctx(pos, BPF_REG_1, args[i].value, + BPF_PROLOGUE_START_ARG_REG + i)) + goto errout; + + if (check_pos(pos)) + goto errout; + return 0; +errout: + return -1; +} + +/* + * Slow path: + * At least one argument has the form of 'offset($rx)'. + * + * Following code first stores them into stack, then loads all of then + * to r2 - r5. + * Before final loading, the final result should be: + * + * low address + * BPF_REG_FP - 24 ARG3 + * BPF_REG_FP - 16 ARG2 + * BPF_REG_FP - 8 ARG1 + * BPF_REG_FP + * high address + * + * For each argument (described as: offn(...off2(off1(reg)))), + * generates following code: + * + * r7 <- fp + * r7 <- r7 - stack_offset // Ideal code should initialize r7 using + * // fp before generating args. However, + * // eBPF won't regard r7 as stack pointer + * // if it is generated by minus 8 from + * // another stack pointer except fp. + * // This is why we have to set r7 + * // to fp for each variable. + * r3 <- value of 'reg'-> generated using gen_ldx_reg_from_ctx() + * (r7) <- r3 // skip following instructions for bare reg + * r3 <- r3 + off1 . // skip if off1 == 0 + * r2 <- 8 \ + * r1 <- r7 |-> generated by gen_read_mem() + * call probe_read / + * jnei r0, 0, err ./ + * r3 <- (r7) + * r3 <- r3 + off2 . // skip if off2 == 0 + * r2 <- 8 \ // r2 may be broken by probe_read, so set again + * r1 <- r7 |-> generated by gen_read_mem() + * call probe_read / + * jnei r0, 0, err ./ + * ... + */ +static int +gen_prologue_slowpath(struct bpf_insn_pos *pos, + struct probe_trace_arg *args, int nargs) +{ + int i; + + for (i = 0; i < nargs; i++) { + struct probe_trace_arg *arg = &args[i]; + const char *reg = arg->value; + struct probe_trace_arg_ref *ref = NULL; + int stack_offset = (i + 1) * -8; + + pr_debug("prologue: fetch arg %d, base reg is %s\n", + i, reg); + + /* value of base register is stored into ARG3 */ + if (gen_ldx_reg_from_ctx(pos, BPF_REG_CTX, reg, + BPF_REG_ARG3)) { + pr_err("prologue: failed to get offset of register %s\n", + reg); + goto errout; + } + + /* Make r7 the stack pointer. */ + ins(BPF_MOV64_REG(BPF_REG_7, BPF_REG_FP), pos); + /* r7 += -8 */ + ins(BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, stack_offset), pos); + /* + * Store r3 (base register) onto stack + * Ensure fp[offset] is set. + * fp is the only valid base register when storing + * into stack. We are not allowed to use r7 as base + * register here. + */ + ins(BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_ARG3, + stack_offset), pos); + + ref = arg->ref; + while (ref) { + pr_debug("prologue: arg %d: offset %ld\n", + i, ref->offset); + if (gen_read_mem(pos, BPF_REG_3, BPF_REG_7, + ref->offset)) { + pr_err("prologue: failed to generate probe_read function call\n"); + goto errout; + } + + ref = ref->next; + /* + * Load previous result into ARG3. Use + * BPF_REG_FP instead of r7 because verifier + * allows FP based addressing only. + */ + if (ref) + ins(BPF_LDX_MEM(BPF_DW, BPF_REG_ARG3, + BPF_REG_FP, stack_offset), pos); + } + } + + /* Final pass: read to registers */ + for (i = 0; i < nargs; i++) + ins(BPF_LDX_MEM(BPF_DW, BPF_PROLOGUE_START_ARG_REG + i, + BPF_REG_FP, -BPF_REG_SIZE * (i + 1)), pos); + + ins(BPF_JMP_IMM(BPF_JA, BPF_REG_0, 0, JMP_TO_SUCCESS_CODE), pos); + + if (check_pos(pos)) + goto errout; + return 0; +errout: + return -1; +} + +static int +prologue_relocate(struct bpf_insn_pos *pos, struct bpf_insn *error_code, + struct bpf_insn *success_code, struct bpf_insn *user_code) +{ + struct bpf_insn *insn; + + if (check_pos(pos)) + return -ERANGE; + + for (insn = pos->begin; insn < pos->pos; insn++) { + u8 class = BPF_CLASS(insn->code); + u8 opcode; + + if (class != BPF_JMP) + continue; + opcode = BPF_OP(insn->code); + if (opcode == BPF_CALL) + continue; + + switch (insn->off) { + case JMP_TO_ERROR_CODE: + insn->off = error_code - (insn + 1); + break; + case JMP_TO_SUCCESS_CODE: + insn->off = success_code - (insn + 1); + break; + case JMP_TO_USER_CODE: + insn->off = user_code - (insn + 1); + break; + default: + pr_err("bpf prologue: internal error: relocation failed\n"); + return -1; + } + } + return 0; +} + +int bpf__gen_prologue(struct probe_trace_arg *args, int nargs, + struct bpf_insn *new_prog, size_t *new_cnt, + size_t cnt_space) +{ + struct bpf_insn *success_code = NULL; + struct bpf_insn *error_code = NULL; + struct bpf_insn *user_code = NULL; + struct bpf_insn_pos pos; + bool fastpath = true; + int i; + + if (!new_prog || !new_cnt) + return -EINVAL; + + pos.begin = new_prog; + pos.end = new_prog + cnt_space; + pos.pos = new_prog; + + if (!nargs) { + ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 0), + &pos); + + if (check_pos(&pos)) + goto errout; + + *new_cnt = pos_get_cnt(&pos); + return 0; + } + + if (nargs > BPF_PROLOGUE_MAX_ARGS) + nargs = BPF_PROLOGUE_MAX_ARGS; + if (cnt_space > BPF_MAXINSNS) + cnt_space = BPF_MAXINSNS; + + /* First pass: validation */ + for (i = 0; i < nargs; i++) { + struct probe_trace_arg_ref *ref = args[i].ref; + + if (args[i].value[0] == '@') { + /* TODO: fetch global variable */ + pr_err("bpf: prologue: global %s%+ld not support\n", + args[i].value, ref ? ref->offset : 0); + return -ENOTSUP; + } + + while (ref) { + /* fastpath is true if all args has ref == NULL */ + fastpath = false; + + /* + * Instruction encodes immediate value using + * s32, ref->offset is long. On systems which + * can't fill long in s32, refuse to process if + * ref->offset too large (or small). + */ +#ifdef __LP64__ +#define OFFSET_MAX ((1LL << 31) - 1) +#define OFFSET_MIN ((1LL << 31) * -1) + if (ref->offset > OFFSET_MAX || + ref->offset < OFFSET_MIN) { + pr_err("bpf: prologue: offset out of bound: %ld\n", + ref->offset); + return -E2BIG; + } +#endif + ref = ref->next; + } + } + pr_debug("prologue: pass validation\n"); + + if (fastpath) { + /* If all variables are registers... */ + pr_debug("prologue: fast path\n"); + if (gen_prologue_fastpath(&pos, args, nargs)) + goto errout; + } else { + pr_debug("prologue: slow path\n"); + + /* Initialization: move ctx to a callee saved register. */ + ins(BPF_MOV64_REG(BPF_REG_CTX, BPF_REG_ARG1), &pos); + + if (gen_prologue_slowpath(&pos, args, nargs)) + goto errout; + /* + * start of ERROR_CODE (only slow pass needs error code) + * mov r2 <- 1 + * goto usercode + */ + error_code = pos.pos; + ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 1), + &pos); + + for (i = 0; i < nargs; i++) + ins(BPF_ALU64_IMM(BPF_MOV, + BPF_PROLOGUE_START_ARG_REG + i, + 0), + &pos); + ins(BPF_JMP_IMM(BPF_JA, BPF_REG_0, 0, JMP_TO_USER_CODE), + &pos); + } + + /* + * start of SUCCESS_CODE: + * mov r2 <- 0 + * goto usercode // skip + */ + success_code = pos.pos; + ins(BPF_ALU64_IMM(BPF_MOV, BPF_PROLOGUE_FETCH_RESULT_REG, 0), &pos); + + /* + * start of USER_CODE: + * Restore ctx to r1 + */ + user_code = pos.pos; + if (!fastpath) { + /* + * Only slow path needs restoring of ctx. In fast path, + * register are loaded directly from r1. + */ + ins(BPF_MOV64_REG(BPF_REG_ARG1, BPF_REG_CTX), &pos); + if (prologue_relocate(&pos, error_code, success_code, + user_code)) + goto errout; + } + + if (check_pos(&pos)) + goto errout; + + *new_cnt = pos_get_cnt(&pos); + return 0; +errout: + return -ERANGE; +} diff --git a/tools/perf/util/bpf-prologue.h b/tools/perf/util/bpf-prologue.h new file mode 100644 index 0000000..f1e4c5d --- /dev/null +++ b/tools/perf/util/bpf-prologue.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2015, He Kuang <heku...@huawei.com> + * Copyright (C) 2015, Huawei Inc. + */ +#ifndef __BPF_PROLOGUE_H +#define __BPF_PROLOGUE_H + +#include <linux/compiler.h> +#include <linux/filter.h> +#include "probe-event.h" + +#define BPF_PROLOGUE_MAX_ARGS 3 +#define BPF_PROLOGUE_START_ARG_REG BPF_REG_3 +#define BPF_PROLOGUE_FETCH_RESULT_REG BPF_REG_2 + +#ifdef HAVE_BPF_PROLOGUE +int bpf__gen_prologue(struct probe_trace_arg *args, int nargs, + struct bpf_insn *new_prog, size_t *new_cnt, + size_t cnt_space); +#else +static inline int +bpf__gen_prologue(struct probe_trace_arg *args __maybe_unused, + int nargs __maybe_unused, + struct bpf_insn *new_prog __maybe_unused, + size_t *new_cnt, + size_t cnt_space __maybe_unused) +{ + if (!new_cnt) + return -EINVAL; + *new_cnt = 0; + return 0; +} +#endif +#endif /* __BPF_PROLOGUE_H */ -- 1.8.3.4 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/