[PATCH RFC 9/9] veth: Avoid per-packet spinlock of XDP napi ring on enqueueing

[Toshiaki Makita]

From: Toshiaki Makita <makita.toshi...@lab.ntt.co.jp>

Use percpu temporary storage to avoid per-packet spinlock.
This is different from dequeue in that multiple veth devices can be
redirect target in one napi loop so allocate percpu storage in veth
private structure.

Signed-off-by: Toshiaki Makita <makita.toshi...@lab.ntt.co.jp>
---
 drivers/net/veth.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 1592119e3873..5978d76f2c00 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -38,12 +38,18 @@ struct pcpu_vstats {
        struct u64_stats_sync   syncp;
 };
 
+struct xdp_queue {
+       void *q[VETH_XDP_QUEUE_SIZE];
+       unsigned int len;
+};
+
 struct veth_priv {
        struct napi_struct      xdp_napi;
        struct net_device       *dev;
        struct bpf_prog __rcu   *xdp_prog;
        struct net_device __rcu *peer;
        atomic64_t              dropped;
+       struct xdp_queue __percpu *xdp_produce_q;
        struct xdp_mem_info     xdp_mem;
        unsigned                requested_headroom;
        bool                    rx_notify_masked;
@@ -147,8 +153,48 @@ static void veth_ptr_free(void *ptr)
        }
 }
 
+static void veth_xdp_cleanup_queues(struct veth_priv *priv)
+{
+       int cpu;
+
+       for_each_possible_cpu(cpu) {
+               struct xdp_queue *q = per_cpu_ptr(priv->xdp_produce_q, cpu);
+               int i;
+
+               for (i = 0; i < q->len; i++)
+                       veth_ptr_free(q->q[i]);
+
+               q->len = 0;
+       }
+}
+
+static bool veth_xdp_flush_queue(struct veth_priv *priv)
+{
+       struct xdp_queue *q = this_cpu_ptr(priv->xdp_produce_q);
+       int i;
+
+       if (unlikely(!q->len))
+               return false;
+
+       spin_lock(&priv->xdp_ring.producer_lock);
+       for (i = 0; i < q->len; i++) {
+               void *ptr = q->q[i];
+
+               if (unlikely(__ptr_ring_produce(&priv->xdp_ring, ptr)))
+                       veth_ptr_free(ptr);
+       }
+       spin_unlock(&priv->xdp_ring.producer_lock);
+
+       q->len = 0;
+
+       return true;
+}
+
 static void __veth_xdp_flush(struct veth_priv *priv)
 {
+       if (unlikely(!veth_xdp_flush_queue(priv)))
+               return;
+
        /* Write ptr_ring before reading rx_notify_masked */
        smp_mb();
        if (!priv->rx_notify_masked) {
@@ -159,9 +205,13 @@ static void __veth_xdp_flush(struct veth_priv *priv)
 
 static int veth_xdp_enqueue(struct veth_priv *priv, void *ptr)
 {
-       if (unlikely(ptr_ring_produce(&priv->xdp_ring, ptr)))
+       struct xdp_queue *q = this_cpu_ptr(priv->xdp_produce_q);
+
+       if (unlikely(q->len >= VETH_XDP_QUEUE_SIZE))
                return -ENOSPC;
 
+       q->q[q->len++] = ptr;
+
        return 0;
 }
 
@@ -644,6 +694,7 @@ static void veth_napi_del(struct net_device *dev)
 
        napi_disable(&priv->xdp_napi);
        netif_napi_del(&priv->xdp_napi);
+       veth_xdp_cleanup_queues(priv);
        ptr_ring_cleanup(&priv->xdp_ring, veth_ptr_free);
 }
 
@@ -711,15 +762,28 @@ static int is_valid_veth_mtu(int mtu)
 
 static int veth_dev_init(struct net_device *dev)
 {
+       struct veth_priv *priv = netdev_priv(dev);
+
        dev->vstats = netdev_alloc_pcpu_stats(struct pcpu_vstats);
        if (!dev->vstats)
                return -ENOMEM;
+
+       priv->xdp_produce_q = __alloc_percpu(sizeof(*priv->xdp_produce_q),
+                                            sizeof (void *));
+       if (!priv->xdp_produce_q) {
+               free_percpu(dev->vstats);
+               return -ENOMEM;
+       }
+
        return 0;
 }
 
 static void veth_dev_free(struct net_device *dev)
 {
+       struct veth_priv *priv = netdev_priv(dev);
+
        free_percpu(dev->vstats);
+       free_percpu(priv->xdp_produce_q);
 }
 
 #ifdef CONFIG_NET_POLL_CONTROLLER
-- 
2.14.3