The BPF_ST_ATOMIC_REG macro token-pasted the legacy rte_atomicNN_*() API names. It also stacked three casts on the destination pointer and reached a 'return 0' out of the macro into the caller's control flow.
Replace it with two small static-inline helpers, bpf_atomic32() and bpf_atomic64(), that dispatch on ins->imm internally and use the C11 atomic intrinsics directly. The destination is cast once, to a properly __rte_atomic-qualified pointer. The helpers return a status and the dispatch loop owns the early exit. Use memory order seq_cst to preserve the previous behavior of rte_atomicNN_add() / rte_atomicNN_exchange() and matches the Linux kernel BPF interpreter for these opcodes. Signed-off-by: Stephen Hemminger <[email protected]> --- lib/bpf/bpf_exec.c | 91 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/lib/bpf/bpf_exec.c b/lib/bpf/bpf_exec.c index 18013753b1..b8116db191 100644 --- a/lib/bpf/bpf_exec.c +++ b/lib/bpf/bpf_exec.c @@ -64,28 +64,6 @@ (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \ (type)(reg)[(ins)->src_reg]) -#define BPF_ST_ATOMIC_REG(reg, ins, tp) do { \ - switch (ins->imm) { \ - case BPF_ATOMIC_ADD: \ - rte_atomic##tp##_add((rte_atomic##tp##_t *) \ - (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \ - (reg)[(ins)->src_reg]); \ - break; \ - case BPF_ATOMIC_XCHG: \ - (reg)[(ins)->src_reg] = rte_atomic##tp##_exchange((uint##tp##_t *) \ - (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \ - (reg)[(ins)->src_reg]); \ - break; \ - default: \ - /* this should be caught by validator and never reach here */ \ - RTE_BPF_LOG_LINE(ERR, \ - "%s(%p): unsupported atomic operation at pc: %#zx;", \ - __func__, bpf, \ - (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \ - return 0; \ - } \ -} while (0) - /* BPF_LD | BPF_ABS/BPF_IND */ #define NOP(x) (x) @@ -105,6 +83,69 @@ reg[EBPF_REG_0] = op(p[0]); \ } while (0) +/* + * Atomic ops on the BPF target memory. + * + * BPF atomic instructions encode the destination as base register + + * signed offset, with the value to combine taken from src_reg. + * + * Memory order: seq_cst preserves the previous behavior of + * rte_atomicNN_add() / rte_atomicNN_exchange() and matches what the + * Linux kernel BPF interpreter does for these opcodes. + * + * Returns 0 on unsupported sub-op (validator should have rejected it), + * 1 otherwise. + */ +static inline int +bpf_atomic32(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM], + const struct ebpf_insn *ins) +{ + /* need to casts to make bpf memory suitable for C11 atomic */ + uint32_t __rte_atomic *dst + = (uint32_t __rte_atomic *)(uintptr_t)(reg[ins->dst_reg] + ins->off); + uint32_t val = (uint32_t)reg[ins->src_reg]; + + switch (ins->imm) { + case BPF_ATOMIC_ADD: + rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst); + return 1; + case BPF_ATOMIC_XCHG: + reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val, + rte_memory_order_seq_cst); + return 1; + default: + RTE_BPF_LOG_LINE(ERR, + "%s(%p): unsupported atomic operation at pc: %#zx;", + __func__, bpf, + (uintptr_t)ins - (uintptr_t)bpf->prm.ins); + return 0; + } +} + +static inline int +bpf_atomic64(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM], + const struct ebpf_insn *ins) +{ + uint64_t __rte_atomic *dst + = (uint64_t __rte_atomic *)(uintptr_t) (reg[ins->dst_reg] + ins->off); + uint64_t val = reg[ins->src_reg]; + + switch (ins->imm) { + case BPF_ATOMIC_ADD: + rte_atomic_fetch_add_explicit(dst, val, rte_memory_order_seq_cst); + return 1; + case BPF_ATOMIC_XCHG: + reg[ins->src_reg] = rte_atomic_exchange_explicit(dst, val, + rte_memory_order_seq_cst); + return 1; + default: + RTE_BPF_LOG_LINE(ERR, + "%s(%p): unsupported atomic operation at pc: %#zx;", + __func__, bpf, + (uintptr_t)ins - (uintptr_t)bpf->prm.ins); + return 0; + } +} static inline void bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins) @@ -392,10 +433,12 @@ bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM]) break; /* atomic instructions */ case (BPF_STX | EBPF_ATOMIC | BPF_W): - BPF_ST_ATOMIC_REG(reg, ins, 32); + if (bpf_atomic32(bpf, reg, ins) == 0) + return 0; break; case (BPF_STX | EBPF_ATOMIC | EBPF_DW): - BPF_ST_ATOMIC_REG(reg, ins, 64); + if (bpf_atomic64(bpf, reg, ins) == 0) + return 0; break; /* jump instructions */ case (BPF_JMP | BPF_JA): -- 2.53.0
