From: Toshiaki Makita <makita.toshi...@lab.ntt.co.jp> This is preparation for XDP TX and ndo_xdp_xmit.
Now the napi ring accepts both skb and xdp_frame. When xdp_frame is enqueued, skb will not be allocated until XDP program on veth returns PASS. This will speedup the XDP processing when ndo_xdp_xmit is implemented and xdp_frame is enqueued by the peer device. Signed-off-by: Toshiaki Makita <makita.toshi...@lab.ntt.co.jp> --- drivers/net/veth.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 452771f31c30..89c91c1c9935 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -27,6 +27,7 @@ #define DRV_NAME "veth" #define DRV_VERSION "1.0" +#define VETH_XDP_FLAG 0x1UL #define VETH_RING_SIZE 256 #define VETH_XDP_HEADROOM (XDP_PACKET_HEADROOM + NET_IP_ALIGN) @@ -48,6 +49,16 @@ struct veth_priv { struct xdp_rxq_info xdp_rxq; }; +static bool veth_is_xdp_frame(void *ptr) +{ + return (unsigned long)ptr & VETH_XDP_FLAG; +} + +static void *veth_ptr_to_xdp(void *ptr) +{ + return (void *)((unsigned long)ptr & ~VETH_XDP_FLAG); +} + /* * ethtool interface */ @@ -117,7 +128,14 @@ static void veth_ptr_free(void *ptr) { if (!ptr) return; - dev_kfree_skb_any(ptr); + + if (veth_is_xdp_frame(ptr)) { + struct xdp_frame *frame = veth_ptr_to_xdp(ptr); + + xdp_return_frame(frame); + } else { + dev_kfree_skb_any(ptr); + } } static void veth_xdp_flush(struct veth_priv *priv) @@ -263,6 +281,60 @@ static struct sk_buff *veth_build_skb(void *head, int headroom, int len, return skb; } +static struct sk_buff *veth_xdp_rcv_one(struct veth_priv *priv, + struct xdp_frame *frame) +{ + struct bpf_prog *xdp_prog; + unsigned int headroom; + struct sk_buff *skb; + int len, delta = 0; + + rcu_read_lock(); + xdp_prog = rcu_dereference(priv->xdp_prog); + if (xdp_prog) { + struct xdp_buff xdp; + u32 act; + + xdp.data_hard_start = frame->data - frame->headroom; + xdp.data = frame->data; + xdp.data_end = frame->data + frame->len; + xdp.data_meta = frame->data - frame->metasize; + xdp.rxq = &priv->xdp_rxq; + + act = bpf_prog_run_xdp(xdp_prog, &xdp); + + switch (act) { + case XDP_PASS: + delta = frame->data - xdp.data; + break; + default: + bpf_warn_invalid_xdp_action(act); + case XDP_ABORTED: + trace_xdp_exception(priv->dev, xdp_prog, act); + case XDP_DROP: + goto err_xdp; + } + } + rcu_read_unlock(); + + headroom = frame->data - delta - (void *)frame; + len = frame->len + delta; + skb = veth_build_skb(frame, headroom, len, 0); + if (!skb) { + xdp_return_frame(frame); + goto err; + } + + skb->protocol = eth_type_trans(skb, priv->dev); +err: + return skb; +err_xdp: + rcu_read_unlock(); + xdp_return_frame(frame); + + return NULL; +} + static struct sk_buff *veth_xdp_rcv_skb(struct veth_priv *priv, struct sk_buff *skb) { @@ -372,7 +444,10 @@ static int veth_xdp_rcv(struct veth_priv *priv, int budget) if (!ptr) break; - skb = veth_xdp_rcv_skb(priv, ptr); + if (veth_is_xdp_frame(ptr)) + skb = veth_xdp_rcv_one(priv, veth_ptr_to_xdp(ptr)); + else + skb = veth_xdp_rcv_skb(priv, ptr); if (skb) napi_gro_receive(&priv->xdp_napi, skb); -- 2.14.3