Accessing a location of guest physical memory involves a translation
from guest physical addresses to host virtual address. This patch
removes unnecessary accesses by virtio implementation, so that at
high processing rates the bottleneck goes from ~1Mtts to ~2Mtts.
Signed-off-by: Vincenzo Maffione <v.maffi...@gmail.com>
---
hw/virtio/virtio.c | 118 +++++++++++++++++++++++++++++++++++++++--------------
1 file changed, 88 insertions(+), 30 deletions(-)
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 1edef59..1f853d6 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -21,6 +21,7 @@
#include "hw/virtio/virtio-bus.h"
#include "migration/migration.h"
#include "hw/virtio/virtio-access.h"
+#include "exec/address-spaces.h"
/*
* The alignment to use between consumer and producer parts of vring.
@@ -71,6 +72,8 @@ struct VirtQueue
{
VRing vring;
uint16_t last_avail_idx;
+ uint16_t last_fetched_avail_idx;
+ uint16_t last_used_idx;
/* Last used index value we have signalled on */
uint16_t signalled_used;
@@ -138,6 +141,15 @@ static inline uint16_t vring_desc_next(VirtIODevice *vdev,
hwaddr desc_pa,
return virtio_lduw_phys(vdev, pa);
}
+static inline void
+vring_desc_fetch(VirtIODevice *vdev, hwaddr desc_pa, int i, VRingDesc *desc)
+{
+ hwaddr pa;
+ pa = desc_pa + sizeof(VRingDesc) * i;
+ address_space_rw(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
+ (uint8_t *)desc, sizeof(VRingDesc), false);
+}
+
static inline uint16_t vring_avail_flags(VirtQueue *vq)
{
hwaddr pa;
@@ -178,6 +190,15 @@ static inline void vring_used_ring_len(VirtQueue *vq, int
i, uint32_t val)
virtio_stl_phys(vq->vdev, pa, val);
}
+static inline void
+vring_used_writeback(VirtQueue *vq, int i, VRingUsedElem *ue)
+{
+ hwaddr pa;
+ pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
+ address_space_rw(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
+ (uint8_t *)ue, sizeof(VRingUsedElem), true);
+}
+
static uint16_t vring_used_idx(VirtQueue *vq)
{
hwaddr pa;
@@ -190,6 +211,7 @@ static inline void vring_used_idx_set(VirtQueue *vq,
uint16_t val)
hwaddr pa;
pa = vq->vring.used + offsetof(VRingUsed, idx);
virtio_stw_phys(vq->vdev, pa, val);
+ vq->last_used_idx = val;
}
static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
@@ -277,15 +299,17 @@ void virtqueue_discard(VirtQueue *vq, const
VirtQueueElement *elem,
void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
unsigned int len, unsigned int idx)
{
+ VRingUsedElem ue;
+
trace_virtqueue_fill(vq, elem, len, idx);
virtqueue_unmap_sg(vq, elem, len);
- idx = (idx + vring_used_idx(vq)) % vq->vring.num;
+ idx = (idx + vq->last_used_idx) % vq->vring.num;
- /* Get a pointer to the next entry in the used ring. */
- vring_used_ring_id(vq, idx, elem->index);
- vring_used_ring_len(vq, idx, len);
+ ue.id = elem->index;
+ ue.len = len;
+ vring_used_writeback(vq, idx, &ue);
}
void virtqueue_flush(VirtQueue *vq, unsigned int count)
@@ -294,7 +318,7 @@ void virtqueue_flush(VirtQueue *vq, unsigned int count)
/* Make sure buffer is written before we update index. */
smp_wmb();
trace_virtqueue_flush(vq, count);
- old = vring_used_idx(vq);
+ old = vq->last_used_idx;
new = old + count;
vring_used_idx_set(vq, new);
vq->inuse -= count;
@@ -328,6 +352,18 @@ static int virtqueue_num_heads(VirtQueue *vq, unsigned int
idx)
return num_heads;
}
+static bool virtqueue_empty(VirtQueue *vq)
+{
+ if (vq->last_avail_idx == vq->last_fetched_avail_idx) {
+ vq->last_fetched_avail_idx = vring_avail_idx(vq);
+ /* On success, callers read a descriptor at vq->last_avail_idx.
+ * Make sure descriptor read does not bypass avail index read. */
+ smp_rmb();
+ }
+
+ return vq->last_avail_idx == vq->last_fetched_avail_idx;
+}
+
static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
{
unsigned int head;
@@ -345,18 +381,18 @@ static unsigned int virtqueue_get_head(VirtQueue *vq,
unsigned int idx)
return head;
}
-static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa,
- unsigned int i, unsigned int max)
+static unsigned virtqueue_next_desc(VirtIODevice *vdev, VRingDesc *desc,
+ unsigned int max)
{
unsigned int next;
/* If this descriptor says it doesn't chain, we're done. */
- if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) {
+ if (!(desc->flags & VRING_DESC_F_NEXT)) {
return max;
}
/* Check they're not leading us off end of descriptors. */
- next = vring_desc_next(vdev, desc_pa, i);
+ next = desc->next;
/* Make sure compiler knows to grab that: we don't want it changing! */
smp_wmb();
@@ -382,15 +418,17 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned
int *in_bytes,
VirtIODevice *vdev = vq->vdev;
unsigned int max, num_bufs, indirect = 0;
hwaddr desc_pa;
+ VRingDesc desc;
int i;
max = vq->vring.num;
num_bufs = total_bufs;
i = virtqueue_get_head(vq, idx++);
desc_pa = vq->vring.desc;
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
- if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
- if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+ if (desc.flags & VRING_DESC_F_INDIRECT) {
+ if (desc.len % sizeof(VRingDesc)) {
error_report("Invalid size for indirect buffer table");
exit(1);
}
@@ -403,27 +441,34 @@ void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned
int *in_bytes,
/* loop over the indirect descriptor table */
indirect = 1;
- max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
- desc_pa = vring_desc_addr(vdev, desc_pa, i);
+ max = desc.len / sizeof(VRingDesc);
+ desc_pa = desc.addr;
num_bufs = i = 0;
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
}
- do {
+ for (;;) {
/* If we've got too many, that implies a descriptor loop. */
if (++num_bufs > max) {
error_report("Looped descriptor");
exit(1);
}
- if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
- in_total += vring_desc_len(vdev, desc_pa, i);
+ if (desc.flags & VRING_DESC_F_WRITE) {
+ in_total += desc.len;
} else {
- out_total += vring_desc_len(vdev, desc_pa, i);
+ out_total += desc.len;
}
if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
goto done;
}
- } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
+
+ i = virtqueue_next_desc(vdev, &desc, max);
+ if (i == max) {
+ break;
+ }
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
+ }
if (!indirect)
total_bufs = num_bufs;
@@ -506,9 +551,11 @@ int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
unsigned int i, head, max;
hwaddr desc_pa = vq->vring.desc;
VirtIODevice *vdev = vq->vdev;
+ VRingDesc desc;
- if (!virtqueue_num_heads(vq, vq->last_avail_idx))
+ if (virtqueue_empty(vq)) {
return 0;
+ }
/* When we start there are none of either input nor output. */
elem->out_num = elem->in_num = 0;
@@ -520,46 +567,55 @@ int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
vring_set_avail_event(vq, vq->last_avail_idx);
}
- if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
- if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
+
+ if (desc.flags & VRING_DESC_F_INDIRECT) {
+ if (desc.len % sizeof(VRingDesc)) {
error_report("Invalid size for indirect buffer table");
exit(1);
}
/* loop over the indirect descriptor table */
- max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
- desc_pa = vring_desc_addr(vdev, desc_pa, i);
+ max = desc.len / sizeof(VRingDesc);
+ desc_pa = desc.addr;
i = 0;
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
}
/* Collect all the descriptors */
- do {
+ for (;;) {
struct iovec *sg;
- if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
+ if (desc.flags & VRING_DESC_F_WRITE) {
if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
error_report("Too many write descriptors in indirect table");
exit(1);
}
- elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i);
+ elem->in_addr[elem->in_num] = desc.addr;
sg = &elem->in_sg[elem->in_num++];
} else {
if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
error_report("Too many read descriptors in indirect table");
exit(1);
}
- elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i);
+ elem->out_addr[elem->out_num] = desc.addr;
sg = &elem->out_sg[elem->out_num++];
}
- sg->iov_len = vring_desc_len(vdev, desc_pa, i);
+ sg->iov_len = desc.len;
/* If we've got too many, that implies a descriptor loop. */
if ((elem->in_num + elem->out_num) > max) {
error_report("Looped descriptor");
exit(1);
}
- } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
+
+ i = virtqueue_next_desc(vdev, &desc, max);
+ if (i == max) {
+ break;
+ }
+ vring_desc_fetch(vdev, desc_pa, i, &desc);
+ }
/* Now map what we have collected */
virtqueue_map(elem);
@@ -673,6 +729,8 @@ void virtio_reset(void *opaque)
vdev->vq[i].vring.avail = 0;
vdev->vq[i].vring.used = 0;
vdev->vq[i].last_avail_idx = 0;
+ vdev->vq[i].last_fetched_avail_idx = 0;
+ vdev->vq[i].last_used_idx = 0;
virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
vdev->vq[i].signalled_used = 0;
vdev->vq[i].signalled_used_valid = false;
@@ -1052,7 +1110,7 @@ static bool vring_notify(VirtIODevice *vdev, VirtQueue
*vq)
v = vq->signalled_used_valid;
vq->signalled_used_valid = true;
old = vq->signalled_used;
- new = vq->signalled_used = vring_used_idx(vq);
+ new = vq->signalled_used = vq->last_used_idx;
return !v || vring_need_event(vring_get_used_event(vq), new, old);
}
--
2.6.3
