Since the atomic operations that are added in subsequent commits are
all isomorphic with BPF_ADD, pull out a macro to avoid the
interpreter becoming dominated by lines of atomic-related code.

Note that this sacrificies interpreter performance (combining
STX_ATOMIC_W and STX_ATOMIC_DW into single switch case means that we
need an extra conditional branch to differentiate them) in favour of
compact and (relatively!) simple C code.

Acked-by: Yonghong Song <y...@fb.com>
Signed-off-by: Brendan Jackman <jackm...@google.com>
---
 kernel/bpf/core.c | 80 +++++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 41 deletions(-)

diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 28f960bc2e30..1d9e5dcde03a 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1618,55 +1618,53 @@ static u64 ___bpf_prog_run(u64 *regs, const struct 
bpf_insn *insn, u64 *stack)
        LDX_PROBE(DW, 8)
 #undef LDX_PROBE
 
-       STX_ATOMIC_W:
-               switch (IMM) {
-               case BPF_ADD:
-                       /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */
-                       atomic_add((u32) SRC, (atomic_t *)(unsigned long)
-                                  (DST + insn->off));
-                       break;
-               case BPF_ADD | BPF_FETCH:
-                       SRC = (u32) atomic_fetch_add(
-                               (u32) SRC,
-                               (atomic_t *)(unsigned long) (DST + insn->off));
-                       break;
-               case BPF_XCHG:
-                       SRC = (u32) atomic_xchg(
-                               (atomic_t *)(unsigned long) (DST + insn->off),
-                               (u32) SRC);
-                       break;
-               case BPF_CMPXCHG:
-                       BPF_R0 = (u32) atomic_cmpxchg(
-                               (atomic_t *)(unsigned long) (DST + insn->off),
-                               (u32) BPF_R0, (u32) SRC);
+#define ATOMIC_ALU_OP(BOP, KOP)                                                
\
+               case BOP:                                               \
+                       if (BPF_SIZE(insn->code) == BPF_W)              \
+                               atomic_##KOP((u32) SRC, (atomic_t *)(unsigned 
long) \
+                                            (DST + insn->off));        \
+                       else                                            \
+                               atomic64_##KOP((u64) SRC, (atomic64_t 
*)(unsigned long) \
+                                              (DST + insn->off));      \
+                       break;                                          \
+               case BOP | BPF_FETCH:                                   \
+                       if (BPF_SIZE(insn->code) == BPF_W)              \
+                               SRC = (u32) atomic_fetch_##KOP(         \
+                                       (u32) SRC,                      \
+                                       (atomic_t *)(unsigned long) (DST + 
insn->off)); \
+                       else                                            \
+                               SRC = (u64) atomic64_fetch_##KOP(       \
+                                       (u64) SRC,                      \
+                                       (atomic64_t *)(s64) (DST + insn->off)); 
\
                        break;
-               default:
-                       goto default_label;
-               }
-               CONT;
 
        STX_ATOMIC_DW:
+       STX_ATOMIC_W:
                switch (IMM) {
-               case BPF_ADD:
-                       /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */
-                       atomic64_add((u64) SRC, (atomic64_t *)(unsigned long)
-                                    (DST + insn->off));
-                       break;
-               case BPF_ADD | BPF_FETCH:
-                       SRC = (u64) atomic64_fetch_add(
-                               (u64) SRC,
-                               (atomic64_t *)(s64) (DST + insn->off));
-                       break;
+               ATOMIC_ALU_OP(BPF_ADD, add)
+#undef ATOMIC_ALU_OP
+
                case BPF_XCHG:
-                       SRC = (u64) atomic64_xchg(
-                               (atomic64_t *)(u64) (DST + insn->off),
-                               (u64) SRC);
+                       if (BPF_SIZE(insn->code) == BPF_W)
+                               SRC = (u32) atomic_xchg(
+                                       (atomic_t *)(unsigned long) (DST + 
insn->off),
+                                       (u32) SRC);
+                       else
+                               SRC = (u64) atomic64_xchg(
+                                       (atomic64_t *)(u64) (DST + insn->off),
+                                       (u64) SRC);
                        break;
                case BPF_CMPXCHG:
-                       BPF_R0 = (u64) atomic64_cmpxchg(
-                               (atomic64_t *)(u64) (DST + insn->off),
-                               (u64) BPF_R0, (u64) SRC);
+                       if (BPF_SIZE(insn->code) == BPF_W)
+                               BPF_R0 = (u32) atomic_cmpxchg(
+                                       (atomic_t *)(unsigned long) (DST + 
insn->off),
+                                       (u32) BPF_R0, (u32) SRC);
+                       else
+                               BPF_R0 = (u64) atomic64_cmpxchg(
+                                       (atomic64_t *)(u64) (DST + insn->off),
+                                       (u64) BPF_R0, (u64) SRC);
                        break;
+
                default:
                        goto default_label;
                }
-- 
2.29.2.576.ga3fc446d84-goog

Reply via email to