Introduce bpf_arch_text_poke_batch() to patch multiple targets in batch, which utilizes smp_text_poke_batch_add/finish() on x86_64.
Thereafter, it will able to text-poke multiple bpf progs at once via bpf_arch_text_poke_batch(). Signed-off-by: Leon Hwang <[email protected]> --- arch/x86/net/bpf_jit_comp.c | 41 +++++++++++++++++++++++++++++++++++++ include/linux/bpf.h | 9 ++++++++ kernel/bpf/core.c | 5 +++++ 3 files changed, 55 insertions(+) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c80d3161d458..b2040ae09273 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -734,6 +734,47 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, return __bpf_arch_text_poke(ip, old_t, new_t, old_addr, new_addr); } +int bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) +{ + u8 old_insn[X86_PATCH_SIZE]; + u8 new_insn[X86_PATCH_SIZE]; + int err; + u32 i; + + for (i = 0; i < cnt; i++) { + err = bpf_arch_text_poke_check_ip(&pokes[i]->ip); + if (err) + return err; + } + + guard(mutex)(&text_mutex); + + for (i = 0; i < cnt; i++) { + struct bpf_text_poke *poke = pokes[i]; + + err = bpf_arch_text_poke_prepare_insns(poke->ip, poke->old_t, poke->new_t, + poke->old_addr, poke->new_addr, old_insn, + new_insn); + if (err) + return err; + if (memcmp(poke->ip, old_insn, X86_PATCH_SIZE)) + return -EBUSY; + } + + for (i = 0; i < cnt; i++) { + struct bpf_text_poke *poke = pokes[i]; + + bpf_arch_text_poke_prepare_insns(poke->ip, poke->old_t, poke->new_t, + poke->old_addr, poke->new_addr, old_insn, + new_insn); + if (memcmp(poke->ip, new_insn, X86_PATCH_SIZE)) + smp_text_poke_batch_add(poke->ip, new_insn, X86_PATCH_SIZE, NULL); + } + + smp_text_poke_batch_finish(); + return 0; +} + #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8) static void __emit_indirect_jump(u8 **pprog, int reg, bool ereg) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 54144372281c..5b1a189e6e89 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4231,6 +4231,15 @@ enum bpf_text_poke_type { BPF_MOD_JUMP, }; +struct bpf_text_poke { + void *ip; + void *old_addr; + void *new_addr; + enum bpf_text_poke_type old_t; + enum bpf_text_poke_type new_t; +}; + +int bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt); int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 05c89396119a..49b794878c11 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3484,6 +3484,11 @@ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, return -EFAULT; } +int __weak bpf_arch_text_poke_batch(struct bpf_text_poke **pokes, u32 cnt) +{ + return -EOPNOTSUPP; +} + int __weak bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) -- 2.55.0
