Allow eBPF programs attached to TC qdiscs call skb_vlan_push/pop
via helper functions. These functions may change skb-data/hlen.
This data is cached by s390 JIT to improve performance of ld_abs/ld_ind
instructions. Therefore after a change we have to reload the data.
In case of usage of skb_vlan_push/pop, in the prologue we store
the SKB pointer on the stack and restore it after BPF_JMP_CALL
to skb_vlan_push/pop.
Signed-off-by: Michael Holzheu holz...@linux.vnet.ibm.com
---
arch/s390/net/bpf_jit.h | 5 +++-
arch/s390/net/bpf_jit_comp.c | 55 ++--
2 files changed, 37 insertions(+), 23 deletions(-)
diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h
index f6498ee..f010c93 100644
--- a/arch/s390/net/bpf_jit.h
+++ b/arch/s390/net/bpf_jit.h
@@ -36,6 +36,8 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
* | BPF stack | |
* | | |
* +---+ |
+ * | 8 byte skbp | |
+ * R15+170 - +---+ |
* | 8 byte hlen | |
* R15+168 - +---+ |
* | 4 byte align | |
@@ -51,11 +53,12 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[];
* We get 160 bytes stack space from calling function, but only use
* 12 * 8 byte for old backchain, r15..r6, and tail_call_cnt.
*/
-#define STK_SPACE (MAX_BPF_STACK + 8 + 4 + 4 + 160)
+#define STK_SPACE (MAX_BPF_STACK + 8 + 8 + 4 + 4 + 160)
#define STK_160_UNUSED (160 - 12 * 8)
#define STK_OFF(STK_SPACE - STK_160_UNUSED)
#define STK_OFF_TMP160 /* Offset of tmp buffer on stack */
#define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */
+#define STK_OFF_SKBP 170 /* Offset of SKB pointer on stack */
#define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */
#define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index a025ddc..ece46d4 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -53,6 +53,7 @@ struct bpf_jit {
#define SEEN_LITERAL 8 /* code uses literals */
#define SEEN_FUNC 16 /* calls C functions */
#define SEEN_TAIL_CALL 32 /* code uses tail calls */
+#define SEEN_SKB_CHANGE64 /* code changes skb data */
#define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB)
/*
@@ -382,6 +383,26 @@ static void save_restore_regs(struct bpf_jit *jit, int op)
}
/*
+ * For SKB access %b1 contains the SKB pointer. For bpf_jit.S
+ * we store the SKB header length on the stack and the SKB data
+ * pointer in REG_SKB_DATA.
+ */
+static void emit_load_skb_data_hlen(struct bpf_jit *jit)
+{
+ /* Header length: llgf %w1,len(%b1) */
+ EMIT6_DISP_LH(0xe300, 0x0016, REG_W1, REG_0, BPF_REG_1,
+ offsetof(struct sk_buff, len));
+ /* s %w1,data_len(%b1) */
+ EMIT4_DISP(0x5b00, REG_W1, BPF_REG_1,
+ offsetof(struct sk_buff, data_len));
+ /* stg %w1,ST_OFF_HLEN(%r0,%r15) */
+ EMIT6_DISP_LH(0xe300, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
+ /* lg %skb_data,data_off(%b1) */
+ EMIT6_DISP_LH(0xe300, 0x0004, REG_SKB_DATA, REG_0,
+ BPF_REG_1, offsetof(struct sk_buff, data));
+}
+
+/*
* Emit function prologue
*
* Save registers and create stack frame if necessary.
@@ -421,25 +442,12 @@ static void bpf_jit_prologue(struct bpf_jit *jit, bool
is_classic)
EMIT6_DISP_LH(0xe300, 0x0024, REG_W1, REG_0,
REG_15, 152);
}
- /*
-* For SKB access %b1 contains the SKB pointer. For bpf_jit.S
-* we store the SKB header length on the stack and the SKB data
-* pointer in REG_SKB_DATA.
-*/
- if (jit-seen SEEN_SKB) {
- /* Header length: llgf %w1,len(%b1) */
- EMIT6_DISP_LH(0xe300, 0x0016, REG_W1, REG_0, BPF_REG_1,
- offsetof(struct sk_buff, len));
- /* s %w1,data_len(%b1) */
- EMIT4_DISP(0x5b00, REG_W1, BPF_REG_1,
- offsetof(struct sk_buff, data_len));
- /* stg %w1,ST_OFF_HLEN(%r0,%r15) */
+ if (jit-seen SEEN_SKB)
+ emit_load_skb_data_hlen(jit);
+ if (jit-seen SEEN_SKB_CHANGE)
+ /* stg %b1,ST_OFF_SKBP(%r0,%r15) */
EMIT6_DISP_LH(0xe300, 0x0024, REG_W1, REG_0, REG_15,
- STK_OFF_HLEN);
- /* lg %skb_data,data_off(%b1) */
- EMIT6_DISP_LH(0xe300, 0x0004, REG_SKB_DATA, REG_0,
- BPF_REG_1, offsetof(struct sk_buff, data));
- }
+ STK_OFF_SKBP);
/* Clear A (%b0) and X (%b7) registers for converted BPF programs */
if
