It was noticed that the copy_user() friends that was used to access
virtqueue metdata tends to be very expensive for dataplane
implementation like vhost since it involves lots of software checks,
speculation barrier, hardware feature toggling (e.g SMAP). The
extra cost will be more obvious when transferring small packets since
the time spent on metadata accessing become significant..
This patch tries to eliminate those overhead by accessing them through
kernel virtual address by vmap(). To make the pages can be migrated,
instead of pinning them through GUP, we use mmu notifiers to
invalidate vmaps and re-establish vmaps during each round of metadata
prefetching in necessary. For devices that doesn't use metadata
prefetch, the memory acessors fallback to normal copy_user()
implementation gracefully.
Note that this was only done when device IOTLB is not enabled. We
could use similar method to optimize it in the future.
Tests shows about ~24% improvement on TX PPS when using virtio-user +
vhost_net + xdp1 on TAP:
Before: ~5.0Mpps
After: ~6.1Mpps
Signed-off-by: Jason Wang
---
drivers/vhost/net.c | 4 +-
drivers/vhost/vhost.c | 259 +-
drivers/vhost/vhost.h | 15 ++-
3 files changed, 271 insertions(+), 7 deletions(-)
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 36f3d0f49e60..0b4b3deab5aa 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -971,7 +971,7 @@ static void handle_tx(struct vhost_net *net)
if (!sock)
goto out;
- if (!vq_iotlb_prefetch(vq))
+ if (!vq_meta_prefetch(vq))
goto out;
vhost_disable_notify(>dev, vq);
@@ -1140,7 +1140,7 @@ static void handle_rx(struct vhost_net *net)
if (!sock)
goto out;
- if (!vq_iotlb_prefetch(vq))
+ if (!vq_meta_prefetch(vq))
goto out;
vhost_disable_notify(>dev, vq);
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 337ce6f5a098..46a889b61a4d 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -440,6 +440,9 @@ void vhost_dev_init(struct vhost_dev *dev,
vq->indirect = NULL;
vq->heads = NULL;
vq->dev = dev;
+ memset(>avail_ring, 0, sizeof(vq->avail_ring));
+ memset(>used_ring, 0, sizeof(vq->used_ring));
+ memset(>desc_ring, 0, sizeof(vq->desc_ring));
mutex_init(>mutex);
vhost_vq_reset(dev, vq);
if (vq->handle_kick)
@@ -489,6 +492,61 @@ bool vhost_dev_has_owner(struct vhost_dev *dev)
}
EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
+static int vhost_invalidate_vmap(struct vhost_virtqueue *vq,
+struct vhost_vmap *map,
+unsigned long uaddr,
+unsigned long start,
+unsigned long end,
+bool blockable)
+{
+ if (start < uaddr && end >= uaddr) {
+ if (!blockable)
+ return -EAGAIN;
+ mutex_lock(>mutex);
+ if (map->addr)
+ vunmap(map->unmap_addr);
+ map->addr = NULL;
+ map->unmap_addr = NULL;
+ mutex_unlock(>mutex);
+ }
+
+ return 0;
+}
+
+static int vhost_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
+struct mm_struct *mm,
+unsigned long start,
+unsigned long end,
+bool blockable)
+{
+ struct vhost_dev *dev = container_of(mn, struct vhost_dev,
+mmu_notifier);
+ int i;
+
+ for (i = 0; i < dev->nvqs; i++) {
+ struct vhost_virtqueue *vq = dev->vqs[i];
+
+ if (vhost_invalidate_vmap(vq, >avail_ring,
+ (unsigned long)vq->avail,
+ start, end, blockable))
+ return -EAGAIN;
+ if (vhost_invalidate_vmap(vq, >desc_ring,
+ (unsigned long)vq->desc,
+ start, end, blockable))
+ return -EAGAIN;
+ if (vhost_invalidate_vmap(vq, >used_ring,
+ (unsigned long)vq->used,
+ start, end, blockable))
+ return -EAGAIN;
+ }
+
+ return 0;
+}
+
+static const struct mmu_notifier_ops vhost_mmu_notifier_ops = {
+ .invalidate_range_start = vhost_mmu_notifier_invalidate_range_start,
+};
+
/* Caller should have device mutex */
long vhost_dev_set_owner(struct vhost_dev *dev)
{
@@ -520,7 +578,14 @@ long
