This makes it possible to batch notifications, and the fewer barriers helps performance:
Before: (./vringh_test --eventidx --parallel): Using CPUS 0 and 3 Guest: notified 156428, pinged 156251 Host: notified 156251, pinged 78215 R=4.575 U=3.520 S=3.520 Using CPUS 0 and 3 Guest: notified 156617, pinged 156250 Host: notified 156250, pinged 78310 R=4.511 U=3.580 S=3.580 Using CPUS 0 and 3 Guest: notified 156357, pinged 156251 Host: notified 156251, pinged 78179 R=4.518 U=3.536 S=3.536 After: Using CPUS 0 and 3 Guest: notified 156394, pinged 156251 Host: notified 156251, pinged 78197 R=4.282 U=3.456 S=3.456 Using CPUS 0 and 3 Guest: notified 156578, pinged 156251 Host: notified 156251, pinged 78289 R=4.248 U=3.436 S=3.436 Using CPUS 0 and 3 Guest: notified 156594, pinged 156251 Host: notified 156251, pinged 78297 R=4.329 U=3.416 S=3.416 Signed-off-by: Rusty Russell <ru...@rustcorp.com.au> --- drivers/vhost/vringh.c | 89 +++++++++++++++++++++++++++++++++--------------- include/linux/vringh.h | 15 +++++--- 2 files changed, 72 insertions(+), 32 deletions(-) diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index b28670f..ab10da8 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -290,13 +290,12 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len, int (*putu16)(u16 *p, u16 val), int (*putused)(struct vring_used_elem *dst, const struct vring_used_elem - *s), - bool *notify) + *s)) { struct vring_used_elem used; struct vring_used *used_ring; int err; - u16 used_idx, old, used_event; + u16 used_idx; used.id = idx; used.len = len; @@ -309,7 +308,7 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len, } used_ring = vrh->vring.used; - used_idx = vrh->last_used_idx; + used_idx = vrh->last_used_idx + vrh->completed; err = putused(&used_ring->ring[used_idx % vrh->vring.num], &used); @@ -323,19 +322,24 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len, /* Make sure buffer is written before we update index. */ virtio_wmb(vrh->weak_barriers); - old = vrh->last_used_idx; - vrh->last_used_idx++; - - err = putu16(&vrh->vring.used->idx, vrh->last_used_idx); + err = putu16(&vrh->vring.used->idx, used_idx + 1); if (err) { vringh_bad("Failed to update used index at %p", &vrh->vring.used->idx); return err; } - /* If we already know we need to notify, skip re-checking */ - if (*notify) - return 0; + vrh->completed++; + return 0; +} + + +static inline int __vringh_need_notify(struct vringh *vrh, + int (*getu16)(u16 *val, const u16 *p)) +{ + bool notify; + u16 used_event; + int err; /* Flush out used index update. This is paired with the * barrier that the Guest executes when enabling @@ -351,21 +355,28 @@ static inline int __vringh_complete(struct vringh *vrh, u16 idx, u32 len, &vrh->vring.avail->flags); return err; } - if (!(flags & VRING_AVAIL_F_NO_INTERRUPT)) - *notify = true; - return 0; + return (!(flags & VRING_AVAIL_F_NO_INTERRUPT)); } - /* Modern: we know where other side is up to. */ + /* Modern: we know when other side wants to know. */ err = getu16(&used_event, &vring_used_event(&vrh->vring)); if (err) { vringh_bad("Failed to get used event idx at %p", &vring_used_event(&vrh->vring)); return err; } - if (vring_need_event(used_event, vrh->last_used_idx, old)) - *notify = true; - return 0; + + /* Just in case we added so many that we wrap. */ + if (unlikely(vrh->completed > 0xffff)) + notify = true; + else + notify = vring_need_event(used_event, + vrh->last_used_idx + vrh->completed, + vrh->last_used_idx); + + vrh->last_used_idx += vrh->completed; + vrh->completed = 0; + return notify; } static inline bool __vringh_notify_enable(struct vringh *vrh, @@ -587,14 +598,14 @@ void vringh_abandon_user(struct vringh *vrh, unsigned int num) * @vrh: the vring. * @head: the head as filled in by vringh_getdesc_user. * @len: the length of data we have written. - * @notify: set if we should notify the other side, otherwise left alone. + * + * You should check vringh_need_notify_user() after one or more calls + * to this function. */ -int vringh_complete_user(struct vringh *vrh, u16 head, u32 len, - bool *notify) +int vringh_complete_user(struct vringh *vrh, u16 head, u32 len) { return __vringh_complete(vrh, head, len, - getu16_user, putu16_user, putused_user, - notify); + getu16_user, putu16_user, putused_user); } /** @@ -621,6 +632,17 @@ void vringh_notify_disable_user(struct vringh *vrh) __vringh_notify_disable(vrh, putu16_user); } +/** + * vringh_need_notify_user - must we tell the other side about used buffers? + * @vrh: the vring we've called vringh_complete_user() on. + * + * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. + */ +int vringh_need_notify_user(struct vringh *vrh) +{ + return __vringh_need_notify(vrh, getu16_user); +} + /* Kernelspace access helpers. */ static inline int getu16_kern(u16 *val, const u16 *p) { @@ -783,14 +805,14 @@ void vringh_abandon_kern(struct vringh *vrh, unsigned int num) * @vrh: the vring. * @head: the head as filled in by vringh_getdesc_kern. * @len: the length of data we have written. - * @notify: set if we should notify the other side, otherwise left alone. + * + * You should check vringh_need_notify_kern() after one or more calls + * to this function. */ -int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len, - bool *notify) +int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len) { return __vringh_complete(vrh, head, len, - getu16_kern, putu16_kern, putused_kern, - notify); + getu16_kern, putu16_kern, putused_kern); } /** @@ -816,3 +838,14 @@ void vringh_notify_disable_kern(struct vringh *vrh) { __vringh_notify_disable(vrh, putu16_kern); } + +/** + * vringh_need_notify_kern - must we tell the other side about used buffers? + * @vrh: the vring we've called vringh_complete_kern() on. + * + * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. + */ +int vringh_need_notify_kern(struct vringh *vrh) +{ + return __vringh_need_notify(vrh, getu16_kern); +} diff --git a/include/linux/vringh.h b/include/linux/vringh.h index 508b5e5..9df86e9 100644 --- a/include/linux/vringh.h +++ b/include/linux/vringh.h @@ -44,6 +44,9 @@ struct vringh { /* Last index we used. */ u16 last_used_idx; + /* How many descriptors we've completed since last need_notify(). */ + u32 completed; + /* The vring (note: it may contain user pointers!) */ struct vring vring; }; @@ -83,13 +86,15 @@ ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len); ssize_t vringh_iov_push_user(struct vringh_iov *wiov, const void *src, size_t len); -/* Mark a descriptor as used. Sets notify if you should fire eventfd. */ -int vringh_complete_user(struct vringh *vrh, u16 head, u32 len, - bool *notify); +/* Mark a descriptor as used. */ +int vringh_complete_user(struct vringh *vrh, u16 head, u32 len); /* Pretend we've never seen descriptor (for easy error handling). */ void vringh_abandon_user(struct vringh *vrh, unsigned int num); +/* Do we need to fire the eventfd to notify the other side? */ +int vringh_need_notify_user(struct vringh *vrh); + /* Helpers for kernelspace vrings. */ int vringh_init_kern(struct vringh *vrh, u32 features, unsigned int num, bool weak_barriers, @@ -107,9 +112,11 @@ ssize_t vringh_iov_pull_kern(struct vringh_iov *riov, void *dst, size_t len); ssize_t vringh_iov_push_user(struct vringh_iov *wiov, const void *src, size_t len); void vringh_abandon_kern(struct vringh *vrh, unsigned int num); -int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len, bool *notify); +int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len); bool vringh_notify_enable_kern(struct vringh *vrh); void vringh_notify_disable_kern(struct vringh *vrh); +int vringh_need_notify_kern(struct vringh *vrh); + #endif /* _LINUX_VRINGH_H */ -- 1.7.10.4 _______________________________________________ Virtualization mailing list Virtualization@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/virtualization