On Sun, Sep 20, 2015 at 04:58:42PM +0300, Marcel Apfelbaum wrote:
> On 09/18/2015 06:10 PM, Yuanhan Liu wrote:
> >All queue pairs, including the default (the first) queue pair,
> >are allocated dynamically, when a vring_call message is received
> >first time for a specific queue pair.
> >
> >This is a refactor work for enabling vhost-user multiple queue;
> >it should not break anything as it does no functional changes:
> >we don't support mq set, so there is only one mq at max.
> >
> >This patch is based on Changchun's patch.
> >
> >Signed-off-by: Yuanhan Liu <yuanhan.liu at linux.intel.com>
> >---
> > lib/librte_vhost/rte_virtio_net.h | 3 +-
> > lib/librte_vhost/vhost_user/virtio-net-user.c | 44 +++++-----
> > lib/librte_vhost/virtio-net.c | 121
> > ++++++++++++++++----------
> > 3 files changed, 102 insertions(+), 66 deletions(-)
> >
> >diff --git a/lib/librte_vhost/rte_virtio_net.h
> >b/lib/librte_vhost/rte_virtio_net.h
> >index e3a21e5..5dd6493 100644
> >--- a/lib/librte_vhost/rte_virtio_net.h
> >+++ b/lib/librte_vhost/rte_virtio_net.h
> >@@ -96,7 +96,7 @@ struct vhost_virtqueue {
> > * Device structure contains all configuration information relating to the
> > device.
> > */
> > struct virtio_net {
> >- struct vhost_virtqueue *virtqueue[VIRTIO_QNUM]; /**< Contains
> >all virtqueue information. */
> >+ struct vhost_virtqueue *virtqueue[VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX];
> >/**< Contains all virtqueue information. */
> > struct virtio_memory *mem; /**< QEMU memory and memory
> > region information. */
> > uint64_t features; /**< Negotiated feature set. */
> > uint64_t protocol_features; /**< Negotiated
> > protocol feature set. */
> >@@ -104,6 +104,7 @@ struct virtio_net {
> > uint32_t flags; /**< Device flags. Only used to
> > check if device is running on data core. */
> > #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ)
> > char ifname[IF_NAME_SZ]; /**< Name of the tap
> > device or socket path. */
> >+ uint32_t virt_qp_nb; /**< number of queue pair we
> >have allocated */
> > void *priv; /**< private context */
> > } __rte_cache_aligned;
> >
> >diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c
> >b/lib/librte_vhost/vhost_user/virtio-net-user.c
> >index 360254e..e83d279 100644
> >--- a/lib/librte_vhost/vhost_user/virtio-net-user.c
> >+++ b/lib/librte_vhost/vhost_user/virtio-net-user.c
> >@@ -206,25 +206,33 @@ err_mmap:
> > }
> >
>
> Hi,
>
> > static int
> >+vq_is_ready(struct vhost_virtqueue *vq)
> >+{
> >+ return vq && vq->desc &&
> >+ vq->kickfd != -1 &&
> >+ vq->callfd != -1;
>
> kickfd and callfd are unsigned
That's probably a bug.
fds are signed, and -1 is what qemu uses to mean "nop".
This comparison will convert -1 to unsigned int so it'll work.
The >= ones below won't work.
I think fd types need to be fixed.
> >+}
> >+
> >+static int
> > virtio_is_ready(struct virtio_net *dev)
> > {
> > struct vhost_virtqueue *rvq, *tvq;
> >+ uint32_t i;
> >
> >- /* mq support in future.*/
> >- rvq = dev->virtqueue[VIRTIO_RXQ];
> >- tvq = dev->virtqueue[VIRTIO_TXQ];
> >- if (rvq && tvq && rvq->desc && tvq->desc &&
> >- (rvq->kickfd != -1) &&
> >- (rvq->callfd != -1) &&
> >- (tvq->kickfd != -1) &&
> >- (tvq->callfd != -1)) {
> >- RTE_LOG(INFO, VHOST_CONFIG,
> >- "virtio is now ready for processing.\n");
> >- return 1;
> >+ for (i = 0; i < dev->virt_qp_nb; i++) {
> >+ rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ];
> >+ tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ];
> >+
> >+ if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) {
> >+ RTE_LOG(INFO, VHOST_CONFIG,
> >+ "virtio is not ready for processing.\n");
> >+ return 0;
> >+ }
> > }
> >+
> > RTE_LOG(INFO, VHOST_CONFIG,
> >- "virtio isn't ready for processing.\n");
> >- return 0;
> >+ "virtio is now ready for processing.\n");
> >+ return 1;
> > }
> >
> > void
> >@@ -290,13 +298,9 @@ user_get_vring_base(struct vhost_device_ctx ctx,
> > * sent and only sent in vhost_vring_stop.
> > * TODO: cleanup the vring, it isn't usable since here.
> > */
> >- if ((dev->virtqueue[VIRTIO_RXQ]->kickfd) >= 0) {
> >- close(dev->virtqueue[VIRTIO_RXQ]->kickfd);
> >- dev->virtqueue[VIRTIO_RXQ]->kickfd = -1;
> >- }
> >- if ((dev->virtqueue[VIRTIO_TXQ]->kickfd) >= 0) {
> >- close(dev->virtqueue[VIRTIO_TXQ]->kickfd);
> >- dev->virtqueue[VIRTIO_TXQ]->kickfd = -1;
> >+ if ((dev->virtqueue[state->index]->kickfd) >= 0) {
>
> always >= 0
>
> >+ close(dev->virtqueue[state->index]->kickfd);
> >+ dev->virtqueue[state->index]->kickfd = -1;
>
> again unsigned
>
> > }
> >
> > return 0;
> >diff --git a/lib/librte_vhost/virtio-net.c b/lib/librte_vhost/virtio-net.c
> >index deac6b9..643a92e 100644
> >--- a/lib/librte_vhost/virtio-net.c
> >+++ b/lib/librte_vhost/virtio-net.c
> >@@ -36,6 +36,7 @@
> > #include <stddef.h>
> > #include <stdint.h>
> > #include <stdlib.h>
> >+#include <assert.h>
> > #include <sys/mman.h>
> > #include <unistd.h>
> > #ifdef RTE_LIBRTE_VHOST_NUMA
> >@@ -178,6 +179,15 @@ add_config_ll_entry(struct virtio_net_config_ll
> >*new_ll_dev)
> >
> > }
> >
> >+static void
> >+cleanup_vq(struct vhost_virtqueue *vq)
> >+{
> >+ if (vq->callfd >= 0)
> >+ close(vq->callfd);
> >+ if (vq->kickfd >= 0)
> >+ close(vq->kickfd);
>
> both always >=0
>
> >+}
> >+
> > /*
> > * Unmap any memory, close any file descriptors and
> > * free any memory owned by a device.
> >@@ -185,6 +195,8 @@ add_config_ll_entry(struct virtio_net_config_ll
> >*new_ll_dev)
> > static void
> > cleanup_device(struct virtio_net *dev)
> > {
> >+ uint32_t i;
> >+
> > /* Unmap QEMU memory file if mapped. */
> > if (dev->mem) {
> > munmap((void *)(uintptr_t)dev->mem->mapped_address,
> >@@ -192,15 +204,10 @@ cleanup_device(struct virtio_net *dev)
> > free(dev->mem);
> > }
> >
> >- /* Close any event notifiers opened by device. */
> >- if (dev->virtqueue[VIRTIO_RXQ]->callfd >= 0)
> >- close(dev->virtqueue[VIRTIO_RXQ]->callfd);
> >- if (dev->virtqueue[VIRTIO_RXQ]->kickfd >= 0)
> >- close(dev->virtqueue[VIRTIO_RXQ]->kickfd);
> >- if (dev->virtqueue[VIRTIO_TXQ]->callfd >= 0)
> >- close(dev->virtqueue[VIRTIO_TXQ]->callfd);
> >- if (dev->virtqueue[VIRTIO_TXQ]->kickfd >= 0)
> >- close(dev->virtqueue[VIRTIO_TXQ]->kickfd);
> >+ for (i = 0; i < dev->virt_qp_nb; i++) {
> >+ cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ]);
> >+ cleanup_vq(dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ]);
> >+ }
> > }
> >
> > /*
> >@@ -209,9 +216,11 @@ cleanup_device(struct virtio_net *dev)
> > static void
> > free_device(struct virtio_net_config_ll *ll_dev)
> > {
> >- /* Free any malloc'd memory */
> >- rte_free(ll_dev->dev.virtqueue[VIRTIO_RXQ]);
> >- rte_free(ll_dev->dev.virtqueue[VIRTIO_TXQ]);
> >+ uint32_t i;
> >+
> >+ for (i = 0; i < ll_dev->dev.virt_qp_nb; i++)
> >+ rte_free(ll_dev->dev.virtqueue[i * VIRTIO_QNUM]);
> >+
> > rte_free(ll_dev);
> > }
> >
> >@@ -244,6 +253,50 @@ rm_config_ll_entry(struct virtio_net_config_ll *ll_dev,
> > }
> > }
> >
> >+static void
> >+init_vring_queue(struct vhost_virtqueue *vq)
> >+{
> >+ memset(vq, 0, sizeof(struct vhost_virtqueue));
> >+
> >+ vq->kickfd = -1;
> >+ vq->callfd = -1;
>
> same here
>
> >+
> >+ /* Backends are set to -1 indicating an inactive device. */
> >+ vq->backend = -1;
> >+}
> >+
> >+static void
> >+init_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx)
> >+{
> >+ init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_RXQ]);
> >+ init_vring_queue(dev->virtqueue[qp_idx * VIRTIO_QNUM + VIRTIO_TXQ]);
> >+}
> >+
> >+static int
> >+alloc_vring_queue_pair(struct virtio_net *dev, uint32_t qp_idx)
> >+{
> >+ struct vhost_virtqueue *virtqueue = NULL;
> >+ uint32_t virt_rx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_RXQ;
> >+ uint32_t virt_tx_q_idx = qp_idx * VIRTIO_QNUM + VIRTIO_TXQ;
> >+
> >+ virtqueue = rte_malloc(NULL,
> >+ sizeof(struct vhost_virtqueue) * VIRTIO_QNUM, 0);
> >+ if (virtqueue == NULL) {
> >+ RTE_LOG(ERR, VHOST_CONFIG,
> >+ "Failed to allocate memory for virt qp:%d.\n", qp_idx);
> >+ return -1;
> >+ }
> >+
> >+ dev->virtqueue[virt_rx_q_idx] = virtqueue;
> >+ dev->virtqueue[virt_tx_q_idx] = virtqueue + VIRTIO_TXQ;
> >+
> >+ init_vring_queue_pair(dev, qp_idx);
> >+
> >+ dev->virt_qp_nb += 1;
> >+
> >+ return 0;
> >+}
> >+
> > /*
> > * Initialise all variables in device structure.
> > */
> >@@ -251,6 +304,7 @@ static void
> > init_device(struct virtio_net *dev)
> > {
> > uint64_t vq_offset;
> >+ uint32_t i;
> >
> > /*
> > * Virtqueues have already been malloced so
> >@@ -261,17 +315,9 @@ init_device(struct virtio_net *dev)
> > /* Set everything to 0. */
> > memset((void *)(uintptr_t)((uint64_t)(uintptr_t)dev + vq_offset), 0,
> > (sizeof(struct virtio_net) - (size_t)vq_offset));
> >- memset(dev->virtqueue[VIRTIO_RXQ], 0, sizeof(struct vhost_virtqueue));
> >- memset(dev->virtqueue[VIRTIO_TXQ], 0, sizeof(struct vhost_virtqueue));
> >
> >- dev->virtqueue[VIRTIO_RXQ]->kickfd = -1;
> >- dev->virtqueue[VIRTIO_RXQ]->callfd = -1;
> >- dev->virtqueue[VIRTIO_TXQ]->kickfd = -1;
> >- dev->virtqueue[VIRTIO_TXQ]->callfd = -1;
> >-
> >- /* Backends are set to -1 indicating an inactive device. */
> >- dev->virtqueue[VIRTIO_RXQ]->backend = VIRTIO_DEV_STOPPED;
> >- dev->virtqueue[VIRTIO_TXQ]->backend = VIRTIO_DEV_STOPPED;
> >+ for (i = 0; i < dev->virt_qp_nb; i++)
> >+ init_vring_queue_pair(dev, i);
> > }
> >
> > /*
> >@@ -283,7 +329,6 @@ static int
> > new_device(struct vhost_device_ctx ctx)
> > {
> > struct virtio_net_config_ll *new_ll_dev;
> >- struct vhost_virtqueue *virtqueue_rx, *virtqueue_tx;
> >
> > /* Setup device and virtqueues. */
> > new_ll_dev = rte_malloc(NULL, sizeof(struct virtio_net_config_ll), 0);
> >@@ -294,28 +339,6 @@ new_device(struct vhost_device_ctx ctx)
> > return -1;
> > }
> >
> >- virtqueue_rx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
> >- if (virtqueue_rx == NULL) {
> >- rte_free(new_ll_dev);
> >- RTE_LOG(ERR, VHOST_CONFIG,
> >- "(%"PRIu64") Failed to allocate memory for rxq.\n",
> >- ctx.fh);
> >- return -1;
> >- }
> >-
> >- virtqueue_tx = rte_malloc(NULL, sizeof(struct vhost_virtqueue), 0);
> >- if (virtqueue_tx == NULL) {
> >- rte_free(virtqueue_rx);
> >- rte_free(new_ll_dev);
> >- RTE_LOG(ERR, VHOST_CONFIG,
> >- "(%"PRIu64") Failed to allocate memory for txq.\n",
> >- ctx.fh);
> >- return -1;
> >- }
> >-
> >- new_ll_dev->dev.virtqueue[VIRTIO_RXQ] = virtqueue_rx;
> >- new_ll_dev->dev.virtqueue[VIRTIO_TXQ] = virtqueue_tx;
> >-
> > /* Initialise device and virtqueues. */
> > init_device(&new_ll_dev->dev);
> >
> >@@ -680,13 +703,21 @@ set_vring_call(struct vhost_device_ctx ctx, struct
> >vhost_vring_file *file)
> > {
> > struct virtio_net *dev;
> > struct vhost_virtqueue *vq;
> >+ uint32_t cur_qp_idx = file->index / VIRTIO_QNUM;
> >
> > dev = get_device(ctx);
> > if (dev == NULL)
> > return -1;
> >
> >+ /* alloc vring queue pair if it is a new queue pair */
> >+ if (cur_qp_idx + 1 > dev->virt_qp_nb) {
> >+ if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0)
> >+ return -1;
> >+ }
> >+
> > /* file->index refers to the queue index. The txq is 1, rxq is 0. */
> > vq = dev->virtqueue[file->index];
> >+ assert(vq != NULL);
> >
> > if (vq->callfd >= 0)
> > close(vq->callfd);
> >
>
>
> I hope I helped,
> Thanks,
> Marcel
>
