Each rewrite in the main loop of bpf_do_misc_fixups() patches the program right away, which costs O(prog->len) per rewrite. Queue them with bpf_patch_list_add() instead and commit them once the loop is done, before the stack of subprogs is initialized for may_goto.
The loop now walks the unpatched program, delta stays 0. A rewrite that fixes up the helper call after queueing it goes on to patch_call_imm with insn pointing to the queued copy of the call, so step insn from the program by index instead of incrementing it. Signed-off-by: Qiliang Yuan <[email protected]> --- kernel/bpf/fixups.c | 228 +++++++++++++++++----------------------------------- 1 file changed, 72 insertions(+), 156 deletions(-) diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 39566f3825108..4b96f4ee9b3d8 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -2099,13 +2099,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2190,13 +2186,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = patch - insn_buf; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2220,13 +2212,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_MOV64_IMM(insn->dst_reg, 0); cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2240,13 +2228,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) return -EFAULT; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2293,13 +2277,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); cnt = patch - insn_buf; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2336,13 +2316,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = may_goto_expand(insn_buf, insn->off, stack_off_cnt, tail, ARRAY_SIZE(tail)); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } else if (bpf_is_may_goto_insn(insn)) { int stack_off = -stack_depth - 8; @@ -2355,13 +2331,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = may_goto_expand(insn_buf, insn->off, stack_off, tail, ARRAY_SIZE(tail)); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2392,13 +2364,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 2; i++; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2413,13 +2381,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (cnt == 0) goto next_insn; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2500,13 +2464,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) map)->index_mask); insn_buf[2] = *insn; cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2534,13 +2494,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[2] = *insn; cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto patch_call_imm; } @@ -2553,13 +2509,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[1] = *insn; cnt = 2; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto patch_call_imm; } @@ -2596,14 +2548,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) if (bpf_map_is_percpu_map(map_ptr->map_type)) prog->jit_required = true; - new_prog = bpf_patch_insn_data(env, i + delta, - insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2679,14 +2626,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) BPF_REG_0, 0); cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, - cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2709,13 +2651,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[0] = BPF_ALU32_REG(BPF_XOR, BPF_REG_0, BPF_REG_0); cnt = 1; #endif - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2728,13 +2666,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0); cnt = 3; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } #endif @@ -2762,13 +2696,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[cnt++] = BPF_JMP_A(1); insn_buf[cnt++] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2794,13 +2724,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 1; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2820,13 +2746,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) cnt = 2; } - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2836,12 +2758,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) /* Load IP address from ctx - 16 */ insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16); - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1); - if (!new_prog) - return -ENOMEM; - - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, 1); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2891,13 +2810,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[10] = BPF_MOV64_IMM(BPF_REG_0, -ENOENT); cnt = 11; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } @@ -2909,13 +2824,9 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0); cnt = 2; - new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); - if (!new_prog) - return -ENOMEM; - - delta += cnt - 1; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; + insn = bpf_patch_list_add(env, i + delta, insn_buf, cnt); + if (IS_ERR(insn)) + return PTR_ERR(insn); goto next_insn; } patch_call_imm: @@ -2946,9 +2857,14 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) stack_depth_extra = 0; } i++; - insn++; + insn = &prog->insnsi[i + delta]; } + ret = bpf_patch_list_commit(env); + if (ret) + return ret; + prog = env->prog; + env->prog->aux->stack_depth = subprogs[0].stack_depth; for (i = 0; i < env->subprog_cnt; i++) { int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1; -- 2.43.0
