On Fri, 29 Dec 2017 18:00:04 +0800
Jason Wang <jasow...@redhat.com> wrote:
> This patch implements XDP transmission for TAP. Since we can't create
> new queues for TAP during XDP set, exist ptr_ring was reused for
> queuing XDP buffers. To differ xdp_buff from sk_buff, TUN_XDP_FLAG
> (0x1ULL) was encoded into lowest bit of xpd_buff pointer during
> ptr_ring_produce, and was decoded during consuming. XDP metadata was
> stored in the headroom of the packet which should work in most of
> cases since driver usually reserve enough headroom. Very minor changes
> were done for vhost_net: it just need to peek the length depends on
> the type of pointer.
>
> Tests was done on two Intel E5-2630 2.40GHz machines connected back to
> back through two 82599ES. Traffic were generated through MoonGen and
> testpmd(rxonly) in guest reports 2.97Mpps when xdp_redirect_map is
> doing redirection from ixgbe to TAP.
IMHO a performance measurement without something to compare against is
useless. What was the performance before?
> Cc: Jesper Dangaard Brouer <bro...@redhat.com>
> Signed-off-by: Jason Wang <jasow...@redhat.com>
> ---
> drivers/net/tun.c | 205
> ++++++++++++++++++++++++++++++++++++++++---------
> drivers/vhost/net.c | 13 +++-
> include/linux/if_tun.h | 17 ++++
> 3 files changed, 197 insertions(+), 38 deletions(-)
>
> diff --git a/drivers/net/tun.c b/drivers/net/tun.c
> index 2c89efe..be6d993 100644
> --- a/drivers/net/tun.c
> +++ b/drivers/net/tun.c
> @@ -240,6 +240,24 @@ struct tun_struct {
> struct tun_steering_prog __rcu *steering_prog;
> };
>
> +bool tun_is_xdp_buff(void *ptr)
> +{
> + return (unsigned long)ptr & TUN_XDP_FLAG;
> +}
> +EXPORT_SYMBOL(tun_is_xdp_buff);
> +
> +void *tun_xdp_to_ptr(void *ptr)
> +{
> + return (void *)((unsigned long)ptr | TUN_XDP_FLAG);
> +}
> +EXPORT_SYMBOL(tun_xdp_to_ptr);
> +
> +void *tun_ptr_to_xdp(void *ptr)
> +{
> + return (void *)((unsigned long)ptr & ~TUN_XDP_FLAG);
> +}
> +EXPORT_SYMBOL(tun_ptr_to_xdp);
> +
> static int tun_napi_receive(struct napi_struct *napi, int budget)
> {
> struct tun_file *tfile = container_of(napi, struct tun_file, napi);
> @@ -630,12 +648,25 @@ static struct tun_struct *tun_enable_queue(struct
> tun_file *tfile)
> return tun;
> }
>
> +static void tun_ptr_free(void *ptr)
> +{
> + if (!ptr)
> + return;
> + if (tun_is_xdp_buff(ptr)) {
> + struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
> +
> + put_page(virt_to_head_page(xdp->data));
(Yet another XDP-free call point, I need to convert to use an accessor
later to transition driver into an xdp_buff return API)
> + } else {
> + __skb_array_destroy_skb(ptr);
> + }
> +}
> +
> static void tun_queue_purge(struct tun_file *tfile)
> {
> - struct sk_buff *skb;
> + void *ptr;
>
> - while ((skb = ptr_ring_consume(&tfile->tx_ring)) != NULL)
> - kfree_skb(skb);
> + while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
> + tun_ptr_free(ptr);
>
> skb_queue_purge(&tfile->sk.sk_write_queue);
> skb_queue_purge(&tfile->sk.sk_error_queue);
> @@ -688,8 +719,7 @@ static void __tun_detach(struct tun_file *tfile, bool
> clean)
> unregister_netdevice(tun->dev);
> }
> if (tun)
> - ptr_ring_cleanup(&tfile->tx_ring,
> - __skb_array_destroy_skb);
> + ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
> sock_put(&tfile->sk);
> }
> }
> @@ -1201,6 +1231,54 @@ static const struct net_device_ops tun_netdev_ops = {
> .ndo_get_stats64 = tun_net_get_stats64,
> };
>
> +static int tun_xdp_xmit(struct net_device *dev, struct xdp_buff *xdp)
> +{
> + struct tun_struct *tun = netdev_priv(dev);
> + struct xdp_buff *buff = xdp->data_hard_start;
> + int headroom = xdp->data - xdp->data_hard_start;
> + struct tun_file *tfile;
> + u32 numqueues;
> + int ret = 0;
> +
> + /* Assure headroom is available and buff is properly aligned */
> + if (unlikely(headroom < sizeof(*xdp) || tun_is_xdp_buff(xdp)))
> + return -ENOSPC;
> +
> + *buff = *xdp;
> +
> + rcu_read_lock();
> +
> + numqueues = READ_ONCE(tun->numqueues);
> + if (!numqueues) {
> + ret = -ENOSPC;
> + goto out;
> + }
> + tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
> + numqueues]);
Several concurrent CPUs can get the same 'tfile'.
> + /* Encode the XDP flag into lowest bit for consumer to differ
> + * XDP buffer from sk_buff.
> + */
> + if (ptr_ring_produce(&tfile->tx_ring, tun_xdp_to_ptr(buff))) {
> + this_cpu_inc(tun->pcpu_stats->tx_dropped);
> + ret = -ENOSPC;
> + }
The ptr_ring_produce() will take a lock per packet, limiting the
performance. (Again a case where I would have liked a bulk API for
ndo_xdp_xmit()).
> +
> +out:
> + rcu_read_unlock();
> + return ret;
> +}
> +
> +static void tun_xdp_flush(struct net_device *dev)
> +{
> + struct tun_struct *tun = netdev_priv(dev);
> + struct tun_file *tfile = tun->tfiles[0];
> +
> + /* Notify and wake up reader process */
> + if (tfile->flags & TUN_FASYNC)
> + kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
> + tfile->socket.sk->sk_data_ready(tfile->socket.sk);
> +}
> +
> static const struct net_device_ops tap_netdev_ops = {
> .ndo_uninit = tun_net_uninit,
> .ndo_open = tun_net_open,
> @@ -1218,6 +1296,8 @@ static const struct net_device_ops tap_netdev_ops = {
> .ndo_set_rx_headroom = tun_set_headroom,
> .ndo_get_stats64 = tun_net_get_stats64,
> .ndo_bpf = tun_xdp,
> + .ndo_xdp_xmit = tun_xdp_xmit,
> + .ndo_xdp_flush = tun_xdp_flush,
> };
>
> static void tun_flow_init(struct tun_struct *tun)
> @@ -1841,6 +1921,40 @@ static ssize_t tun_chr_write_iter(struct kiocb *iocb,
> struct iov_iter *from)
> return result;
> }
>
> +static ssize_t tun_put_user_xdp(struct tun_struct *tun,
> + struct tun_file *tfile,
> + struct xdp_buff *xdp,
> + struct iov_iter *iter)
> +{
> + int vnet_hdr_sz = 0;
> + size_t size = xdp->data_end - xdp->data;
> + struct tun_pcpu_stats *stats;
> + size_t ret;
> +
> + if (tun->flags & IFF_VNET_HDR) {
> + struct virtio_net_hdr gso = { 0 };
> +
> + vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
> + if (unlikely(iov_iter_count(iter) < vnet_hdr_sz))
> + return -EINVAL;
> + if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) !=
> + sizeof(gso)))
> + return -EFAULT;
> + iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso));
> + }
> +
> + ret = copy_to_iter(xdp->data, size, iter) + vnet_hdr_sz;
> +
> + stats = get_cpu_ptr(tun->pcpu_stats);
> + u64_stats_update_begin(&stats->syncp);
> + stats->tx_packets++;
> + stats->tx_bytes += ret;
> + u64_stats_update_end(&stats->syncp);
> + put_cpu_ptr(tun->pcpu_stats);
> +
> + return ret;
> +}
> +
> /* Put packet to the user space buffer */
> static ssize_t tun_put_user(struct tun_struct *tun,
> struct tun_file *tfile,
[...]
> error = -ERESTARTSYS;
> @@ -1977,36 +2090,42 @@ static struct sk_buff *tun_ring_recv(struct tun_file
> *tfile, int noblock,
>
> out:
> *err = error;
> - return skb;
> + return ptr;
> }
>
> static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
> struct iov_iter *to,
> - int noblock, struct sk_buff *skb)
> + int noblock, void *ptr)
> {
> ssize_t ret;
> int err;
>
> tun_debug(KERN_INFO, tun, "tun_do_read\n");
>
> - if (!iov_iter_count(to)) {
> - if (skb)
> - kfree_skb(skb);
> - return 0;
> - }
> + if (!iov_iter_count(to))
> + tun_ptr_free(ptr);
>
> - if (!skb) {
> + if (!ptr) {
> /* Read frames from ring */
> - skb = tun_ring_recv(tfile, noblock, &err);
> - if (!skb)
> + ptr = tun_ring_recv(tfile, noblock, &err);
> + if (!ptr)
> return err;
> }
>
> - ret = tun_put_user(tun, tfile, skb, to);
> - if (unlikely(ret < 0))
> - kfree_skb(skb);
> - else
> - consume_skb(skb);
> + if (tun_is_xdp_buff(ptr)) {
> + struct xdp_buff *xdp = tun_ptr_to_xdp(ptr);
> +
> + ret = tun_put_user_xdp(tun, tfile, xdp, to);
> + put_page(virt_to_head_page(xdp->data));
This again tie us info a page-refcnt based XDP scheme.
> + } else {
> + struct sk_buff *skb = ptr;
> +
> + ret = tun_put_user(tun, tfile, skb, to);
> + if (unlikely(ret < 0))
> + kfree_skb(skb);
> + else
> + consume_skb(skb);
> + }
>
> return ret;
> }
[...]
> @@ -3191,8 +3323,7 @@ struct socket *tun_get_socket(struct file *file)
> struct tun_file *tfile;
> if (file->f_op != &tun_fops)
> return ERR_PTR(-EINVAL);
> - tfile = file->private_data;
> - if (!tfile)
> + tfile = file->private_data; if (!tfile)
This change looks like a typo...
> return ERR_PTR(-EBADFD);
> return &tfile->socket;
> }
--
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
LinkedIn: http://www.linkedin.com/in/brouer
