[PATCH] eal: allow both allow and block options coexistence
Currently, all buses use the same eal allow and block options. Need to
allow both allow and block options for different buses to coexist.
It wouldn't be a problem for pci bus if both allow and block options
were present. When the first option occurs, the scan mode for pci bus is
set.
For example:
--allow :05:00.0 --block wq0.0
only pci device :05:00.0 will be scanned
all devices except wq0.0 on dsa bus will be scanned
--allow :05:00.0 --block :05:00.1
block option will be ignored
--block :05:00.1 --allow :05:00.0
allow option will be ignored
Signed-off-by: Marvin Liu
---
.mailmap| 2 +-
lib/eal/common/eal_common_options.c | 14 --
2 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/.mailmap b/.mailmap
index 0443e461a9..d256696485 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1548,7 +1548,7 @@ Yogesh Jangra
Yogev Chaimovich
Yongjie Gu
Yongji Xie
-Yong Liu
+Marvin Liu
Yongping Zhang
Yongseok Koh
Yong Wang
diff --git a/lib/eal/common/eal_common_options.c
b/lib/eal/common/eal_common_options.c
index 0305933698..0d8f9c5a38 100644
--- a/lib/eal/common/eal_common_options.c
+++ b/lib/eal/common/eal_common_options.c
@@ -1623,24 +1623,15 @@ int
eal_parse_common_option(int opt, const char *optarg,
struct internal_config *conf)
{
- static int b_used;
- static int a_used;
-
switch (opt) {
case 'b':
- if (a_used)
- goto ba_conflict;
if (eal_option_device_add(RTE_DEVTYPE_BLOCKED, optarg) < 0)
return -1;
- b_used = 1;
break;
case 'a':
- if (b_used)
- goto ba_conflict;
if (eal_option_device_add(RTE_DEVTYPE_ALLOWED, optarg) < 0)
return -1;
- a_used = 1;
break;
/* coremask */
case 'c': {
@@ -1929,11 +1920,6 @@ eal_parse_common_option(int opt, const char *optarg,
}
return 0;
-
-ba_conflict:
- RTE_LOG(ERR, EAL,
- "Options allow (-a) and block (-b) can't be used at the same
time\n");
- return -1;
}
static void
--
2.24.0.rc1
[PATCH] eal: allow both allow and block options coexistence
Since all buses share the same eal allow and block options. There's a
need to allow both allow and block options coexist. At the same time,
this wouldn't be a problem if both allow and block options were present
on the same bus. The scan mode of a bus will be set when met the first
type option.
eg: --allow :05:00.0 --block wq0.0
only pci device :05:00.0 will be scanned
all devices except wq0.0 on dsa bus will be scanned
eg: --allow :05:00.0 --block :05:00.1
block option will be ingored
eg: --block :05:00.1 --allow :05:00.0
allow option will be ignored
Signed-off-by: Marvin Liu
---
.mailmap| 2 +-
lib/eal/common/eal_common_options.c | 14 --
2 files changed, 1 insertion(+), 15 deletions(-)
diff --git a/.mailmap b/.mailmap
index 0443e461a9..d256696485 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1548,7 +1548,7 @@ Yogesh Jangra
Yogev Chaimovich
Yongjie Gu
Yongji Xie
-Yong Liu
+Marvin Liu
Yongping Zhang
Yongseok Koh
Yong Wang
diff --git a/lib/eal/common/eal_common_options.c
b/lib/eal/common/eal_common_options.c
index 0305933698..0d8f9c5a38 100644
--- a/lib/eal/common/eal_common_options.c
+++ b/lib/eal/common/eal_common_options.c
@@ -1623,24 +1623,15 @@ int
eal_parse_common_option(int opt, const char *optarg,
struct internal_config *conf)
{
- static int b_used;
- static int a_used;
-
switch (opt) {
case 'b':
- if (a_used)
- goto ba_conflict;
if (eal_option_device_add(RTE_DEVTYPE_BLOCKED, optarg) < 0)
return -1;
- b_used = 1;
break;
case 'a':
- if (b_used)
- goto ba_conflict;
if (eal_option_device_add(RTE_DEVTYPE_ALLOWED, optarg) < 0)
return -1;
- a_used = 1;
break;
/* coremask */
case 'c': {
@@ -1929,11 +1920,6 @@ eal_parse_common_option(int opt, const char *optarg,
}
return 0;
-
-ba_conflict:
- RTE_LOG(ERR, EAL,
- "Options allow (-a) and block (-b) can't be used at the same
time\n");
- return -1;
}
static void
--
2.24.0.rc1
[PATCH] net/virtio: fix incorrect slots number when indirect feature on
Virtio driver only occupies one slot for enqueuing chained mbufs when
indirect feature is on. Required slots calculation should depend on
indirect feature status at the end.
Fixes: 0eaf7fc2fe8e ("net/virtio: separate AVX Rx/Tx")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx_packed.h
b/drivers/net/virtio/virtio_rxtx_packed.h
index d5c259a1f6..536112983c 100644
--- a/drivers/net/virtio/virtio_rxtx_packed.h
+++ b/drivers/net/virtio/virtio_rxtx_packed.h
@@ -125,13 +125,12 @@ virtqueue_enqueue_single_packed_vec(struct virtnet_tx
*txvq,
* any_layout => number of segments
* default=> number of segments + 1
*/
- slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
can_push = rte_mbuf_refcnt_read(txm) == 1 &&
RTE_MBUF_DIRECT(txm) &&
txm->nb_segs == 1 &&
rte_pktmbuf_headroom(txm) >= hdr_size;
- slots = txm->nb_segs + !can_push;
+ slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
need = slots - vq->vq_free_cnt;
/* Positive value indicates it need free vring descriptors */
--
2.17.1
[dpdk-dev] [PATCH] net/virtio: fix vectorized path receive oversized packets
If packed ring size is not power of two, it is possible that remained
number less than one batch and meanwhile batch operation can pass.
This will cause incorrect remained number calculation and then lead to
receiving oversized packets. The patch fixed the issue by added
remained number check before batch operation.
Fixes: 77d66da83834 ("net/virtio: add vectorized packed ring Rx")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx_packed.c
b/drivers/net/virtio/virtio_rxtx_packed.c
index ab489a58af..45cf39df22 100644
--- a/drivers/net/virtio/virtio_rxtx_packed.c
+++ b/drivers/net/virtio/virtio_rxtx_packed.c
@@ -95,11 +95,13 @@ virtio_recv_pkts_packed_vec(void *rx_queue,
num = num - ((vq->vq_used_cons_idx + num) % PACKED_BATCH_SIZE);
while (num) {
- if (!virtqueue_dequeue_batch_packed_vec(rxvq,
- &rx_pkts[nb_rx])) {
- nb_rx += PACKED_BATCH_SIZE;
- num -= PACKED_BATCH_SIZE;
- continue;
+ if (num >= PACKED_BATCH_SIZE) {
+ if (!virtqueue_dequeue_batch_packed_vec(rxvq,
+ &rx_pkts[nb_rx])) {
+ nb_rx += PACKED_BATCH_SIZE;
+ num -= PACKED_BATCH_SIZE;
+ continue;
+ }
}
if (!virtqueue_dequeue_single_packed_vec(rxvq,
&rx_pkts[nb_rx])) {
--
2.17.1
[dpdk-dev] [PATCH 2/2] vhost: fix async enqueue accessing uninitialized variables
This patch fixs coverity issue in async enqueue function by adding
initialization step before using temporary virtio header.
Coverity issue: 366123
Fixes: cd6760da1076 ("vhost: introduce async enqueue for split ring")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 48b013a9b4..ff39878609 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -986,9 +986,10 @@ async_mbuf_to_desc(struct virtio_net *dev, struct
vhost_virtqueue *vq,
hdr_mbuf = m;
hdr_addr = buf_addr;
- if (unlikely(buf_len < dev->vhost_hlen))
+ if (unlikely(buf_len < dev->vhost_hlen)) {
+ memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));
hdr = &tmp_hdr;
- else
+ } else
hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
--
2.17.1
[dpdk-dev] [PATCH 1/2] vhost: fix accessing uninitialized variables
This patch fixs coverity issue by adding initialization step before
using temporary virtio header.
Coverity issue: 366181
Fixes: fb3815cc614d ("vhost: handle virtually non-contiguous buffers in Rx-mrg")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 7f621fb6dd..48b013a9b4 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -812,9 +812,10 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct
vhost_virtqueue *vq,
hdr_mbuf = m;
hdr_addr = buf_addr;
- if (unlikely(buf_len < dev->vhost_hlen))
+ if (unlikely(buf_len < dev->vhost_hlen)) {
+ memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));
hdr = &tmp_hdr;
- else
+ } else
hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
--
2.17.1
[dpdk-dev] [PATCH 3/3] vhost: fix potential buffer overflow when batch dequeue
Similar as single dequeue, the multiple accesses of descriptor length
will lead to potential risk. One-time access of descriptor length can
eliminate this risk.
Fixes: 75ed51697820 ("vhost: add packed ring batch dequeue")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index de43686522..0a7d008a91 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -2318,7 +2318,7 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
}
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
- pkts[i]->pkt_len = descs[avail_idx + i].len - buf_offset;
+ pkts[i]->pkt_len = lens[i] - buf_offset;
pkts[i]->data_len = pkts[i]->pkt_len;
ids[i] = descs[avail_idx + i].id;
}
--
2.17.1
[dpdk-dev] [PATCH 2/3] vhost: fix packed ring potential buffer overflow
Similar as split ring, the multiple accesses of descriptor length will
lead to potential risk. One-time access of descriptor length can
eliminate this risk.
Fixes: 2f3225a7d69b ("vhost: add vector filling support for packed ring")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 576a0a20c0..de43686522 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -669,9 +669,10 @@ fill_vec_buf_packed_indirect(struct virtio_net *dev,
return -1;
}
- *len += descs[i].len;
+ dlen = descs[i].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[i].addr, descs[i].len,
+ descs[i].addr, dlen,
perm)))
return -1;
}
@@ -692,6 +693,7 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
bool wrap_counter = vq->avail_wrap_counter;
struct vring_packed_desc *descs = vq->desc_packed;
uint16_t vec_id = *vec_idx;
+ uint64_t dlen;
if (avail_idx < vq->last_avail_idx)
wrap_counter ^= 1;
@@ -724,11 +726,12 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
len, perm) < 0))
return -1;
} else {
- *len += descs[avail_idx].len;
+ dlen = descs[avail_idx].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
descs[avail_idx].addr,
- descs[avail_idx].len,
+ dlen,
perm)))
return -1;
}
--
2.17.1
[dpdk-dev] [PATCH 1/3] vhost: fix split ring potential buffer overflow
In vhost datapath, descriptor's length are mostly used in two coherent
operations. First step is used for address translation, second step is
used for memory transaction from guest to host. But the iterval between
two steps will give a window for malicious guest, in which can change
descriptor length after vhost calcuated buffer size. Thus may lead to
buffer overflow in vhost side. This potential risk can be eliminated by
accessing the descriptor length once.
Fixes: 1be4ebb1c464 ("vhost: support indirect descriptor in mergeable Rx")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 583bf379c6..576a0a20c0 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -548,10 +548,11 @@ fill_vec_buf_split(struct virtio_net *dev, struct
vhost_virtqueue *vq,
return -1;
}
- len += descs[idx].len;
+ dlen = descs[idx].len;
+ len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[idx].addr, descs[idx].len,
+ descs[idx].addr, dlen,
perm))) {
free_ind_table(idesc);
return -1;
--
2.17.1
[dpdk-dev] [PATCH 3/3] vhost: fix potential buffer overflow when batch dequeue
Similar as single dequeue, the multiple accesses of descriptor length
will lead to potential risk. One-time access of descriptor length can
eliminate this risk.
Fixes: 75ed51697820 ("vhost: add packed ring batch dequeue")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index de43686522..0a7d008a91 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -2318,7 +2318,7 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
}
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
- pkts[i]->pkt_len = descs[avail_idx + i].len - buf_offset;
+ pkts[i]->pkt_len = lens[i] - buf_offset;
pkts[i]->data_len = pkts[i]->pkt_len;
ids[i] = descs[avail_idx + i].id;
}
--
2.17.1
[dpdk-dev] [PATCH 1/3] vhost: fix split ring potential buffer overflow
In vhost datapath, descriptor's length are mostly used in two coherent
operations. First step is used for address translation, second step is
used for memory transaction from guest to host. But the iterval between
two steps will give a window for malicious guest, in which can change
descriptor length after vhost calcuated buffer size. Thus may lead to
buffer overflow in vhost side. This potential risk can be eliminated by
accessing the descriptor length once.
Fixes: 1be4ebb1c464 ("vhost: support indirect descriptor in mergeable Rx")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 583bf379c6..576a0a20c0 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -548,10 +548,11 @@ fill_vec_buf_split(struct virtio_net *dev, struct
vhost_virtqueue *vq,
return -1;
}
- len += descs[idx].len;
+ dlen = descs[idx].len;
+ len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[idx].addr, descs[idx].len,
+ descs[idx].addr, dlen,
perm))) {
free_ind_table(idesc);
return -1;
--
2.17.1
[dpdk-dev] [PATCH 2/3] vhost: fix packed ring potential buffer overflow
Similar as split ring, the multiple accesses of descriptor length will
lead to potential risk. One-time access of descriptor length can
eliminate this risk.
Fixes: 2f3225a7d69b ("vhost: add vector filling support for packed ring")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 576a0a20c0..de43686522 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -669,9 +669,10 @@ fill_vec_buf_packed_indirect(struct virtio_net *dev,
return -1;
}
- *len += descs[i].len;
+ dlen = descs[i].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[i].addr, descs[i].len,
+ descs[i].addr, dlen,
perm)))
return -1;
}
@@ -692,6 +693,7 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
bool wrap_counter = vq->avail_wrap_counter;
struct vring_packed_desc *descs = vq->desc_packed;
uint16_t vec_id = *vec_idx;
+ uint64_t dlen;
if (avail_idx < vq->last_avail_idx)
wrap_counter ^= 1;
@@ -724,11 +726,12 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
len, perm) < 0))
return -1;
} else {
- *len += descs[avail_idx].len;
+ dlen = descs[avail_idx].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
descs[avail_idx].addr,
- descs[avail_idx].len,
+ dlen,
perm)))
return -1;
}
--
2.17.1
[dpdk-dev] [PATCH] vhost: fix accessing uninitialized variables
This patch fixs coverity issue by adding initialization step before
using temporary virtio header.
Coverity issue: 366181, 366123
Fixes: fb3815cc614d ("vhost: handle virtually non-contiguous buffers in Rx-mrg")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 583bf379c6..fe464b3088 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -808,9 +808,10 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct
vhost_virtqueue *vq,
hdr_mbuf = m;
hdr_addr = buf_addr;
- if (unlikely(buf_len < dev->vhost_hlen))
+ if (unlikely(buf_len < dev->vhost_hlen)) {
+ memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf));
hdr = &tmp_hdr;
- else
+ } else
hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr;
VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n",
--
2.17.1
[dpdk-dev] [PATCH] vhost: fix potential buffer overflow
In vhost datapath, descriptor's length are mostly used in two coherent
operations. First step is used for address translation, second step is
used for memory transaction from guest to host. But the iterval between
two steps will give a window for malicious guest, in which can change
descriptor length after vhost calcuated buffer size. Thus may lead to
buffer overflow in vhost side. This potential risk can be eliminated by
accessing the descriptor length once.
Fixes: 1be4ebb1c464 ("vhost: support indirect descriptor in mergeable Rx")
Fixes: 2f3225a7d69b ("vhost: add vector filling support for packed ring")
Fixes: 75ed51697820 ("vhost: add packed ring batch dequeue")
Signed-off-by: Marvin Liu
Cc: sta...@dpdk.org
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 583bf379c6..0a7d008a91 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -548,10 +548,11 @@ fill_vec_buf_split(struct virtio_net *dev, struct
vhost_virtqueue *vq,
return -1;
}
- len += descs[idx].len;
+ dlen = descs[idx].len;
+ len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[idx].addr, descs[idx].len,
+ descs[idx].addr, dlen,
perm))) {
free_ind_table(idesc);
return -1;
@@ -668,9 +669,10 @@ fill_vec_buf_packed_indirect(struct virtio_net *dev,
return -1;
}
- *len += descs[i].len;
+ dlen = descs[i].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
- descs[i].addr, descs[i].len,
+ descs[i].addr, dlen,
perm)))
return -1;
}
@@ -691,6 +693,7 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
bool wrap_counter = vq->avail_wrap_counter;
struct vring_packed_desc *descs = vq->desc_packed;
uint16_t vec_id = *vec_idx;
+ uint64_t dlen;
if (avail_idx < vq->last_avail_idx)
wrap_counter ^= 1;
@@ -723,11 +726,12 @@ fill_vec_buf_packed(struct virtio_net *dev, struct
vhost_virtqueue *vq,
len, perm) < 0))
return -1;
} else {
- *len += descs[avail_idx].len;
+ dlen = descs[avail_idx].len;
+ *len += dlen;
if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id,
descs[avail_idx].addr,
- descs[avail_idx].len,
+ dlen,
perm)))
return -1;
}
@@ -2314,7 +2318,7 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
}
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
- pkts[i]->pkt_len = descs[avail_idx + i].len - buf_offset;
+ pkts[i]->pkt_len = lens[i] - buf_offset;
pkts[i]->data_len = pkts[i]->pkt_len;
ids[i] = descs[avail_idx + i].id;
}
--
2.17.1
[dpdk-dev] [PATCH] vhost: fix packed ring dequeue offloading
When vhost doing dequeue offloading, will parse ethernet and l3/l4
header of the packet. Then vhost will set corresponded value in mbuf
attributes. Thus mean offloading action should after packet data copy.
Fixes: 75ed51697820 ("vhost: add packed ring batch dequeue")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 730b92e478..0a7d008a91 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -2267,7 +2267,6 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
{
bool wrap = vq->avail_wrap_counter;
struct vring_packed_desc *descs = vq->desc_packed;
- struct virtio_net_hdr *hdr;
uint64_t lens[PACKED_BATCH_SIZE];
uint64_t buf_lens[PACKED_BATCH_SIZE];
uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
@@ -2324,13 +2323,6 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
ids[i] = descs[avail_idx + i].id;
}
- if (virtio_net_with_host_offload(dev)) {
- vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
- hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
- vhost_dequeue_offload(hdr, pkts[i]);
- }
- }
-
return 0;
free_buf:
@@ -2348,6 +2340,7 @@ virtio_dev_tx_batch_packed(struct virtio_net *dev,
{
uint16_t avail_idx = vq->last_avail_idx;
uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf);
+ struct virtio_net_hdr *hdr;
uintptr_t desc_addrs[PACKED_BATCH_SIZE];
uint16_t ids[PACKED_BATCH_SIZE];
uint16_t i;
@@ -2364,6 +2357,13 @@ virtio_dev_tx_batch_packed(struct virtio_net *dev,
(void *)(uintptr_t)(desc_addrs[i] + buf_offset),
pkts[i]->pkt_len);
+ if (virtio_net_with_host_offload(dev)) {
+ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ hdr = (struct virtio_net_hdr *)(desc_addrs[i]);
+ vhost_dequeue_offload(hdr, pkts[i]);
+ }
+ }
+
if (virtio_net_is_inorder(dev))
vhost_shadow_dequeue_batch_packed_inorder(vq,
ids[PACKED_BATCH_SIZE - 1]);
--
2.17.1
[dpdk-dev] [PATCH] doc: clarify disclosure time slot when no response
Sometimes security team won't send confirmation mail back to reporter in three business days. This mean reported vulnerability is either low severity or not a real vulnerability. Reporter should assume that the issue need shortest embargo. After that reporter can submit it through normal bugzilla process or send out fix patch to public. Signed-off-by: Marvin Liu Signed-off-by: Qian Xu diff --git a/doc/guides/contributing/vulnerability.rst b/doc/guides/contributing/vulnerability.rst index b6300252ad..cda814fa69 100644 --- a/doc/guides/contributing/vulnerability.rst +++ b/doc/guides/contributing/vulnerability.rst @@ -99,6 +99,11 @@ Following information must be included in the mail: * Reporter credit * Bug ID (empty and restricted for future reference) +If no confirmation mail send back to reporter in this period, thus mean security +team take this vulnerability as low severity. Furthermore shortest embargo **two weeks** +is required for it. Reporter can sumbit the bug through normal process or send +out patch to public. + CVE Request --- -- 2.17.1
[dpdk-dev] [PATCH] net/virtio: enable packet data prefetch on x86
Data prefetch instruction can preload data into cpu’s hierarchical
cache before data access. Virtio datapath utilized this feature for
data access acceleration. As config RTE_PMD_PACKET_PREFETCH was
discarded, now packet data prefetch is enabled based on architecture.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 42c4c9882..0196290a5 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -106,7 +106,7 @@ virtqueue_store_flags_packed(struct vring_packed_desc *dp,
dp->flags = flags;
}
}
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while(0)
--
2.17.1
[dpdk-dev] [PATCH v3] drivers/net: enable packet data prefetch on x86
Data prefetch instruction can preload data into cpu’s hierarchical
cache before data access. Data paths like e1000 and virtio utilized
this feature for packet data access acceleration. Enabled packet data
prefetch on x86 architecture, as prefetch instruction has been
supported from very early generation.
Signed-off-by: Marvin Liu
---
v3: replace build config with pre-defined architecture macro
v2: move define from meson.build to rte_config.h
---
diff --git a/drivers/net/e1000/em_rxtx.c b/drivers/net/e1000/em_rxtx.c
index 19e3bffd46..5a9cd04c9c 100644
--- a/drivers/net/e1000/em_rxtx.c
+++ b/drivers/net/e1000/em_rxtx.c
@@ -185,7 +185,7 @@ struct em_tx_queue {
#define rte_em_prefetch(p) do {} while(0)
#endif
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while(0)
diff --git a/drivers/net/e1000/igb_rxtx.c b/drivers/net/e1000/igb_rxtx.c
index dd520cd82c..45d4ee0b92 100644
--- a/drivers/net/e1000/igb_rxtx.c
+++ b/drivers/net/e1000/igb_rxtx.c
@@ -198,7 +198,7 @@ struct igb_tx_queue {
#define rte_igb_prefetch(p)do {} while(0)
#endif
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while(0)
diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 6a8718c086..671c32038b 100644
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -25,7 +25,7 @@
#define rte_enic_prefetch(p) do {} while (0)
#endif
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while (0)
diff --git a/drivers/net/fm10k/fm10k_rxtx.c b/drivers/net/fm10k/fm10k_rxtx.c
index 4accaa2cd6..4bc3afadd1 100644
--- a/drivers/net/fm10k/fm10k_rxtx.c
+++ b/drivers/net/fm10k/fm10k_rxtx.c
@@ -10,7 +10,7 @@
#include "fm10k.h"
#include "base/fm10k_type.h"
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while (0)
diff --git a/drivers/net/igc/igc_txrx.c b/drivers/net/igc/igc_txrx.c
index 4654ec41f0..b8b504738e 100644
--- a/drivers/net/igc/igc_txrx.c
+++ b/drivers/net/igc/igc_txrx.c
@@ -16,7 +16,7 @@
#define rte_igc_prefetch(p)do {} while (0)
#endif
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while (0)
diff --git a/drivers/net/ixgbe/ixgbe_rxtx.h b/drivers/net/ixgbe/ixgbe_rxtx.h
index 6d2f7c9da3..4d39de9531 100644
--- a/drivers/net/ixgbe/ixgbe_rxtx.h
+++ b/drivers/net/ixgbe/ixgbe_rxtx.h
@@ -41,7 +41,7 @@
#define RX_RING_SZ ((IXGBE_MAX_RING_DESC + RTE_PMD_IXGBE_RX_MAX_BURST) * \
sizeof(union ixgbe_adv_rx_desc))
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while(0)
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 42c4c9882f..0196290a5d 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -106,7 +106,7 @@ virtqueue_store_flags_packed(struct vring_packed_desc *dp,
dp->flags = flags;
}
}
-#ifdef RTE_PMD_PACKET_PREFETCH
+#if defined(RTE_ARCH_X86)
#define rte_packet_prefetch(p) rte_prefetch1(p)
#else
#define rte_packet_prefetch(p) do {} while(0)
--
2.17.1
[dpdk-dev] [PATCH] net/virtio: fix invalid indirect desc length
When transmitting indirect descriptors, first desc will store net_hdr
and following descs will be mapped to mbuf segments. Total desc number
will be seg_num plus one. Meaning of variable needed is the number of
used descs in packed ring. This value will always be two for indirect
desc. Now use mbuf segments number for calculating correct desc length.
Fixes: de8b3d238074 ("net/virtio: fix indirect descs in packed datapaths")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 8c8ab9889..42c4c9882 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -698,6 +698,7 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq,
struct rte_mbuf *cookie,
struct virtio_net_hdr *hdr;
uint16_t prev;
bool prepend_header = false;
+ uint16_t seg_num = cookie->nb_segs;
id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
@@ -732,7 +733,7 @@ virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq,
struct rte_mbuf *cookie,
*/
start_dp[idx].addr = txvq->virtio_net_hdr_mem +
RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr);
- start_dp[idx].len = (needed + 1) *
+ start_dp[idx].len = (seg_num + 1) *
sizeof(struct vring_packed_desc);
/* reset flags for indirect desc */
head_flags = VRING_DESC_F_INDIRECT;
--
2.17.1
[dpdk-dev] [PATCH v3 5/5] vhost: add packed ring vectorized enqueue
Optimize vhost packed ring enqueue path with SIMD instructions. Four
descriptors status and length are batched handled with AVX512
instructions. Address translation operations are also accelerated
by AVX512 instructions.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index b270c424b..84dc289e9 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1110,5 +1110,9 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
uintptr_t *desc_addrs,
uint16_t *ids);
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts);
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 3bc6b9b20..3e49c88ac 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -1354,6 +1354,22 @@ virtio_dev_rx_single_packed(struct virtio_net *dev,
return 0;
}
+static __rte_always_inline int
+virtio_dev_rx_handle_batch_packed(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts)
+
+{
+#ifdef CC_AVX512_SUPPORT
+ if (unlikely(dev->vectorized))
+ return virtio_dev_rx_batch_packed_avx(dev, vq, pkts);
+ else
+ return virtio_dev_rx_batch_packed(dev, vq, pkts);
+#else
+ return virtio_dev_rx_batch_packed(dev, vq, pkts);
+#endif
+}
+
static __rte_noinline uint32_t
virtio_dev_rx_packed(struct virtio_net *dev,
struct vhost_virtqueue *__rte_restrict vq,
@@ -1367,8 +1383,8 @@ virtio_dev_rx_packed(struct virtio_net *dev,
rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]);
if (remained >= PACKED_BATCH_SIZE) {
- if (!virtio_dev_rx_batch_packed(dev, vq,
- &pkts[pkt_idx])) {
+ if (!virtio_dev_rx_handle_batch_packed(dev, vq,
+ &pkts[pkt_idx])) {
pkt_idx += PACKED_BATCH_SIZE;
remained -= PACKED_BATCH_SIZE;
continue;
diff --git a/lib/librte_vhost/virtio_net_avx.c
b/lib/librte_vhost/virtio_net_avx.c
index e10b2a285..aa47b15ae 100644
--- a/lib/librte_vhost/virtio_net_avx.c
+++ b/lib/librte_vhost/virtio_net_avx.c
@@ -35,9 +35,15 @@
#define PACKED_AVAIL_FLAG ((0ULL | VRING_DESC_F_AVAIL) << FLAGS_BITS_OFFSET)
#define PACKED_AVAIL_FLAG_WRAP ((0ULL | VRING_DESC_F_USED) << \
FLAGS_BITS_OFFSET)
+#define PACKED_WRITE_AVAIL_FLAG (PACKED_AVAIL_FLAG | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
+#define PACKED_WRITE_AVAIL_FLAG_WRAP (PACKED_AVAIL_FLAG_WRAP | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
#define DESC_FLAGS_POS 0xaa
#define MBUF_LENS_POS 0x
+#define DESC_LENS_POS 0x
+#define DESC_LENS_FLAGS_POS 0xB0B0B0B0
int
vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
@@ -182,3 +188,157 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
return -1;
}
+
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts)
+{
+ struct vring_packed_desc *descs = vq->desc_packed;
+ uint16_t avail_idx = vq->last_avail_idx;
+ uint64_t desc_addrs[PACKED_BATCH_SIZE];
+ uint32_t buf_offset = dev->vhost_hlen;
+ uint32_t desc_status;
+ uint64_t lens[PACKED_BATCH_SIZE];
+ uint16_t i;
+ void *desc_addr;
+ uint8_t cmp_low, cmp_high, cmp_result;
+
+ if (unlikely(avail_idx & PACKED_BATCH_MASK))
+ return -1;
+ if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size))
+ return -1;
+
+ /* check refcnt and nb_segs */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+
+ /* load four mbufs rearm data */
+ __m256i mbufs = _mm256_set_epi64x(
+ *pkts[3]->rearm_data,
+ *pkts[2]->rearm_data,
+ *pkts[1]->rearm_data,
+ *pkts[0]->rearm_data);
+
+ uint16_t cmp = _mm256_cmpneq_epu16_mask(mbufs, mbuf_ref);
+ if (cmp & MBUF_LENS_POS)
+ return -1;
+
+ /* check desc status */
+ desc_addr = &vq->desc_packed[avail_idx];
+ __m512i desc_vec = _mm512_loadu_si512(desc_addr);
+
+ __m512i avail_flag_vec;
+ __m512i used_flag_vec;
+ if (vq->avail_wrap_counter) {
+#if defined(RTE_ARCH_I686)
+ avail_flag_vec = _mm512_set4_epi64(PACKE
[dpdk-dev] [PATCH v3 2/5] vhost: reuse packed ring functions
Move parse_ethernet, offload, extbuf functions to header file. These
functions will be reused by vhost vectorized path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 87583c0b6..12b7699cf 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -20,6 +20,10 @@
#include
#include
+#include
+#include
+#include
+#include
#include "rte_vhost.h"
#include "rte_vdpa.h"
#include "rte_vdpa_dev.h"
@@ -878,4 +882,214 @@ mbuf_is_consumed(struct rte_mbuf *m)
return true;
}
+static __rte_always_inline bool
+virtio_net_is_inorder(struct virtio_net *dev)
+{
+ return dev->features & (1ULL << VIRTIO_F_IN_ORDER);
+}
+
+static void
+parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ void *l3_hdr = NULL;
+ struct rte_ether_hdr *eth_hdr;
+ uint16_t ethertype;
+
+ eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
+
+ m->l2_len = sizeof(struct rte_ether_hdr);
+ ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
+
+ if (ethertype == RTE_ETHER_TYPE_VLAN) {
+ struct rte_vlan_hdr *vlan_hdr =
+ (struct rte_vlan_hdr *)(eth_hdr + 1);
+
+ m->l2_len += sizeof(struct rte_vlan_hdr);
+ ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
+ }
+
+ l3_hdr = (char *)eth_hdr + m->l2_len;
+
+ switch (ethertype) {
+ case RTE_ETHER_TYPE_IPV4:
+ ipv4_hdr = l3_hdr;
+ *l4_proto = ipv4_hdr->next_proto_id;
+ m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV4;
+ break;
+ case RTE_ETHER_TYPE_IPV6:
+ ipv6_hdr = l3_hdr;
+ *l4_proto = ipv6_hdr->proto;
+ m->l3_len = sizeof(struct rte_ipv6_hdr);
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV6;
+ break;
+ default:
+ m->l3_len = 0;
+ *l4_proto = 0;
+ *l4_hdr = NULL;
+ break;
+ }
+}
+
+static inline bool
+virtio_net_with_host_offload(struct virtio_net *dev)
+{
+ if (dev->features &
+ ((1ULL << VIRTIO_NET_F_CSUM) |
+(1ULL << VIRTIO_NET_F_HOST_ECN) |
+(1ULL << VIRTIO_NET_F_HOST_TSO4) |
+(1ULL << VIRTIO_NET_F_HOST_TSO6) |
+(1ULL << VIRTIO_NET_F_HOST_UFO)))
+ return true;
+
+ return false;
+}
+
+static __rte_always_inline void
+vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
+{
+ uint16_t l4_proto = 0;
+ void *l4_hdr = NULL;
+ struct rte_tcp_hdr *tcp_hdr = NULL;
+
+ if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
+ return;
+
+ parse_ethernet(m, &l4_proto, &l4_hdr);
+ if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+ if (hdr->csum_start == (m->l2_len + m->l3_len)) {
+ switch (hdr->csum_offset) {
+ case (offsetof(struct rte_tcp_hdr, cksum)):
+ if (l4_proto == IPPROTO_TCP)
+ m->ol_flags |= PKT_TX_TCP_CKSUM;
+ break;
+ case (offsetof(struct rte_udp_hdr, dgram_cksum)):
+ if (l4_proto == IPPROTO_UDP)
+ m->ol_flags |= PKT_TX_UDP_CKSUM;
+ break;
+ case (offsetof(struct rte_sctp_hdr, cksum)):
+ if (l4_proto == IPPROTO_SCTP)
+ m->ol_flags |= PKT_TX_SCTP_CKSUM;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+ switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+ case VIRTIO_NET_HDR_GSO_TCPV4:
+ case VIRTIO_NET_HDR_GSO_TCPV6:
+ tcp_hdr = l4_hdr;
+ m->ol_flags |= PKT_TX_TCP_SEG;
+ m->tso_segsz = hdr->gso_size;
+ m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
+ break;
+ case VIRTIO_NET_HDR_GSO_UDP:
+ m->ol_flags |= PKT_TX_UDP_SEG;
+ m->tso_segsz = hdr->gso_size;
+
[dpdk-dev] [PATCH v3 4/5] vhost: add packed ring vectorized dequeue
Optimize vhost packed ring dequeue path with SIMD instructions. Four
descriptors status check and writeback are batched handled with AVX512
instructions. Address translation operations are also accelerated by
AVX512 instructions.
If platform or compiler not support vectorization, will fallback to
default path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/meson.build b/lib/librte_vhost/meson.build
index cc9aa65c6..5eadcbae4 100644
--- a/lib/librte_vhost/meson.build
+++ b/lib/librte_vhost/meson.build
@@ -8,6 +8,22 @@ endif
if has_libnuma == 1
dpdk_conf.set10('RTE_LIBRTE_VHOST_NUMA', true)
endif
+
+if arch_subdir == 'x86'
+if not machine_args.contains('-mno-avx512f')
+if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+cflags += ['-DCC_AVX512_SUPPORT']
+vhost_avx512_lib = static_library('vhost_avx512_lib',
+ 'virtio_net_avx.c',
+ dependencies: [static_rte_eal,
static_rte_mempool,
+ static_rte_mbuf,
static_rte_ethdev, static_rte_net],
+ include_directories: includes,
+ c_args: [cflags, '-mavx512f',
'-mavx512bw', '-mavx512vl'])
+objs +=
vhost_avx512_lib.extract_objects('virtio_net_avx.c')
+endif
+endif
+endif
+
if (toolchain == 'gcc' and cc.version().version_compare('>=8.3.0'))
cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
elif (toolchain == 'clang' and cc.version().version_compare('>=3.7.0'))
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index a19fe9423..b270c424b 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1100,4 +1100,15 @@ virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct
rte_mempool *mp,
return NULL;
}
+
+int
+vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
+struct vhost_virtqueue *vq,
+struct rte_mempool *mbuf_pool,
+struct rte_mbuf **pkts,
+uint16_t avail_idx,
+uintptr_t *desc_addrs,
+uint16_t *ids);
+
+
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c
index 9757ed053..3bc6b9b20 100644
--- a/lib/librte_vhost/virtio_net.c
+++ b/lib/librte_vhost/virtio_net.c
@@ -2136,6 +2136,28 @@ vhost_reserve_avail_batch_packed(struct virtio_net *dev,
return -1;
}
+static __rte_always_inline int
+vhost_handle_avail_batch_packed(struct virtio_net *dev,
+struct vhost_virtqueue *vq,
+struct rte_mempool *mbuf_pool,
+struct rte_mbuf **pkts,
+uint16_t avail_idx,
+uintptr_t *desc_addrs,
+uint16_t *ids)
+{
+#ifdef CC_AVX512_SUPPORT
+ if (unlikely(dev->vectorized))
+ return vhost_reserve_avail_batch_packed_avx(dev, vq, mbuf_pool,
+ pkts, avail_idx, desc_addrs, ids);
+ else
+ return vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool,
+ pkts, avail_idx, desc_addrs, ids);
+#else
+ return vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool, pkts,
+ avail_idx, desc_addrs, ids);
+#endif
+}
+
static __rte_always_inline int
virtio_dev_tx_batch_packed(struct virtio_net *dev,
struct vhost_virtqueue *vq,
@@ -2148,8 +2170,9 @@ virtio_dev_tx_batch_packed(struct virtio_net *dev,
uint16_t ids[PACKED_BATCH_SIZE];
uint16_t i;
- if (vhost_reserve_avail_batch_packed(dev, vq, mbuf_pool, pkts,
-avail_idx, desc_addrs, ids))
+
+ if (vhost_handle_avail_batch_packed(dev, vq, mbuf_pool, pkts,
+ avail_idx, desc_addrs, ids))
return -1;
vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE)
diff --git a/lib/librte_vhost/virtio_net_avx.c
b/lib/librte_vhost/virtio_net_avx.c
new file mode 100644
index 0..e10b2a285
--- /dev/null
+++ b/lib/librte_vhost/virtio_net_avx.c
@@ -0,0 +1,184 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2016 Intel Corporation
+ */
+#include
+
+#include "vhost.h"
+
+#define BYTE_SIZE 8
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+
[dpdk-dev] [PATCH v3 3/5] vhost: prepare memory regions addresses
Prepare memory regions guest physical addresses for vectorized data
path. These information will be utilized by SIMD instructions to find
matched region index.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 12b7699cf..a19fe9423 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -52,6 +52,8 @@
#define ASYNC_MAX_POLL_SEG 255
+#define MAX_NREGIONS 8
+
#define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2)
#define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2)
@@ -349,6 +351,11 @@ struct inflight_mem_info {
uint64_tsize;
};
+struct mem_regions_range {
+ uint64_t regions_low_addrs[MAX_NREGIONS];
+ uint64_t regions_high_addrs[MAX_NREGIONS];
+};
+
/**
* Device structure contains all configuration information relating
* to the device.
@@ -356,6 +363,7 @@ struct inflight_mem_info {
struct virtio_net {
/* Frontend (QEMU) memory and memory region information */
struct rte_vhost_memory *mem;
+ struct mem_regions_range *regions_range;
uint64_tfeatures;
uint64_tprotocol_features;
int vid;
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 4deceb3e0..2d2a2a1a3 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -185,6 +185,11 @@ vhost_backend_cleanup(struct virtio_net *dev)
dev->inflight_info = NULL;
}
+ if (dev->regions_range) {
+ free(dev->regions_range);
+ dev->regions_range = NULL;
+ }
+
if (dev->slave_req_fd >= 0) {
close(dev->slave_req_fd);
dev->slave_req_fd = -1;
@@ -1230,6 +1235,27 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
}
}
+ RTE_BUILD_BUG_ON(VHOST_MEMORY_MAX_NREGIONS != 8);
+ if (dev->vectorized) {
+ if (dev->regions_range == NULL) {
+ dev->regions_range = calloc(1,
+ sizeof(struct mem_regions_range));
+ if (!dev->regions_range) {
+ VHOST_LOG_CONFIG(ERR,
+ "failed to alloc dev vectorized
area\n");
+ return RTE_VHOST_MSG_RESULT_ERR;
+ }
+ }
+
+ for (i = 0; i < memory->nregions; i++) {
+ dev->regions_range->regions_low_addrs[i] =
+ memory->regions[i].guest_phys_addr;
+ dev->regions_range->regions_high_addrs[i] =
+ memory->regions[i].guest_phys_addr +
+ memory->regions[i].memory_size;
+ }
+ }
+
for (i = 0; i < dev->nr_vring; i++) {
struct vhost_virtqueue *vq = dev->virtqueue[i];
--
2.17.1
[dpdk-dev] [PATCH v3 0/5] vhost add vectorized data path
Packed ring format is imported since virtio spec 1.1. All descriptors are compacted into one single ring when packed ring format is on. It is straight forward that ring operations can be accelerated by utilizing SIMD instructions. This patch set will introduce vectorized data path in vhost library. If vectorized option is on, operations like descs check, descs writeback, address translation will be accelerated by SIMD instructions. On skylake server, it can bring 6% performance gain in loopback case and around 4% performance gain in PvP case. Vhost application can choose whether using vectorized acceleration, just like external buffer feature. If platform or ring format not support vectorized function, vhost will fallback to use default batch function. There will be no impact in current data path. v3: * rename vectorized datapath file * eliminate the impact when avx512 disabled * dynamically allocate memory regions structure * remove unlikely hint for in_order v2: * add vIOMMU support * add dequeue offloading * rebase code Marvin Liu (5): vhost: add vectorized data path vhost: reuse packed ring functions vhost: prepare memory regions addresses vhost: add packed ring vectorized dequeue vhost: add packed ring vectorized enqueue doc/guides/nics/vhost.rst | 5 + doc/guides/prog_guide/vhost_lib.rst | 12 + drivers/net/vhost/rte_eth_vhost.c | 17 +- lib/librte_vhost/meson.build| 16 ++ lib/librte_vhost/rte_vhost.h| 1 + lib/librte_vhost/socket.c | 5 + lib/librte_vhost/vhost.c| 11 + lib/librte_vhost/vhost.h| 239 +++ lib/librte_vhost/vhost_user.c | 26 +++ lib/librte_vhost/virtio_net.c | 258 - lib/librte_vhost/virtio_net_avx.c | 344 11 files changed, 718 insertions(+), 216 deletions(-) create mode 100644 lib/librte_vhost/virtio_net_avx.c -- 2.17.1
[dpdk-dev] [PATCH v3 1/5] vhost: add vectorized data path
Packed ring operations are split into batch and single functions for
performance perspective. Ring operations in batch function can be
accelerated by SIMD instructions like AVX512.
So introduce vectorized parameter in vhost. Vectorized data path can be
selected if platform and ring format matched requirements. Otherwise
will fallback to original data path.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/vhost.rst b/doc/guides/nics/vhost.rst
index d36f3120b..efdaf4de0 100644
--- a/doc/guides/nics/vhost.rst
+++ b/doc/guides/nics/vhost.rst
@@ -64,6 +64,11 @@ The user can specify below arguments in `--vdev` option.
It is used to enable external buffer support in vhost library.
(Default: 0 (disabled))
+#. ``vectorized``:
+
+It is used to enable vectorized data path support in vhost library.
+(Default: 0 (disabled))
+
Vhost PMD event handling
diff --git a/doc/guides/prog_guide/vhost_lib.rst
b/doc/guides/prog_guide/vhost_lib.rst
index ba4c62aeb..5ef3844a0 100644
--- a/doc/guides/prog_guide/vhost_lib.rst
+++ b/doc/guides/prog_guide/vhost_lib.rst
@@ -118,6 +118,18 @@ The following is an overview of some key Vhost API
functions:
It is disabled by default.
+ - ``RTE_VHOST_USER_VECTORIZED``
+Vectorized data path will used when this flag is set. When packed ring
+enabled, available descriptors are stored from frontend driver in sequence.
+SIMD instructions like AVX can be used to handle multiple descriptors
+simultaneously. Thus can accelerate the throughput of ring operations.
+
+* Only packed ring has vectorized data path.
+
+* Will fallback to normal datapath if no vectorization support.
+
+It is disabled by default.
+
* ``rte_vhost_driver_set_features(path, features)``
This function sets the feature bits the vhost-user driver supports. The
diff --git a/drivers/net/vhost/rte_eth_vhost.c
b/drivers/net/vhost/rte_eth_vhost.c
index 66efecb32..8f71054ad 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -34,6 +34,7 @@ enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
#define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso"
#define ETH_VHOST_LINEAR_BUF "linear-buffer"
#define ETH_VHOST_EXT_BUF "ext-buffer"
+#define ETH_VHOST_VECTORIZED "vectorized"
#define VHOST_MAX_PKT_BURST 32
static const char *valid_arguments[] = {
@@ -45,6 +46,7 @@ static const char *valid_arguments[] = {
ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
ETH_VHOST_LINEAR_BUF,
ETH_VHOST_EXT_BUF,
+ ETH_VHOST_VECTORIZED,
NULL
};
@@ -1509,6 +1511,7 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
int tso = 0;
int linear_buf = 0;
int ext_buf = 0;
+ int vectorized = 0;
struct rte_eth_dev *eth_dev;
const char *name = rte_vdev_device_name(dev);
@@ -1618,6 +1621,17 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
flags |= RTE_VHOST_USER_EXTBUF_SUPPORT;
}
+ if (rte_kvargs_count(kvlist, ETH_VHOST_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ ETH_VHOST_VECTORIZED,
+ &open_int, &vectorized);
+ if (ret < 0)
+ goto out_free;
+
+ if (vectorized == 1)
+ flags |= RTE_VHOST_USER_VECTORIZED;
+ }
+
if (dev->device.numa_node == SOCKET_ID_ANY)
dev->device.numa_node = rte_socket_id();
@@ -1666,4 +1680,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_vhost,
"postcopy-support=<0|1> "
"tso=<0|1> "
"linear-buffer=<0|1> "
- "ext-buffer=<0|1>");
+ "ext-buffer=<0|1> "
+ "vectorized=<0|1>");
diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
index 010f16086..c49c1aca2 100644
--- a/lib/librte_vhost/rte_vhost.h
+++ b/lib/librte_vhost/rte_vhost.h
@@ -36,6 +36,7 @@ extern "C" {
/* support only linear buffers (no chained mbufs) */
#define RTE_VHOST_USER_LINEARBUF_SUPPORT (1ULL << 6)
#define RTE_VHOST_USER_ASYNC_COPY (1ULL << 7)
+#define RTE_VHOST_USER_VECTORIZED (1ULL << 8)
/* Features. */
#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 0169d3648..e492c8c87 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -42,6 +42,7 @@ struct vhost_user_socket {
bool extbuf;
bool linearbuf;
bool async_copy;
+ bool vectorized;
/*
* The "supported_features" indicates the feature bits the
@@ -241,6 +242,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket
*vsocket)
dev->async_copy = 1;
}
+ if (vsocket->
[dpdk-dev] [PATCH 2/2] net/virtio: use indirect ring in packed datapath
Like split ring, packed ring will utilize indirect ring elements when
queuing mbufs need multiple descriptors. Thus each packet will take only
one slot when having multiple segments.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index f915b8a2c..b3b1586a7 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -1756,7 +1756,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
struct rte_mbuf *txm = tx_pkts[nb_tx];
- int can_push = 0, slots, need;
+ int can_push = 0, use_indirect = 0, slots, need;
/* optimize ring usage */
if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
@@ -1768,12 +1768,15 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
__alignof__(struct virtio_net_hdr_mrg_rxbuf)))
can_push = 1;
-
+ else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
+txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
+ use_indirect = 1;
/* How many main ring entries are needed to this Tx?
+* indirect => 1
* any_layout => number of segments
* default=> number of segments + 1
*/
- slots = txm->nb_segs + !can_push;
+ slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
need = slots - vq->vq_free_cnt;
/* Positive value indicates it need free vring descriptors */
@@ -1791,7 +1794,8 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
if (can_push)
virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order);
else
- virtqueue_enqueue_xmit_packed(txvq, txm, slots, 0,
+ virtqueue_enqueue_xmit_packed(txvq, txm, slots,
+ use_indirect, 0,
in_order);
virtio_update_packet_stats(&txvq->stats, txm);
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 6a8214725..ce035b574 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -207,19 +207,26 @@ virtqueue_enqueue_single_packed_vec(struct virtnet_tx
*txvq,
struct virtqueue *vq = txvq->vq;
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
- uint16_t slots, can_push;
+ uint16_t slots, can_push = 0, use_indirect = 0;
int16_t need;
+ /* optimize ring usage */
+ if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
+ vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
+ rte_mbuf_refcnt_read(txm) == 1 &&
+ RTE_MBUF_DIRECT(txm) &&
+ txm->nb_segs == 1 &&
+ rte_pktmbuf_headroom(txm) >= hdr_size)
+ can_push = 1;
+ else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
+txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
+ use_indirect = 1;
/* How many main ring entries are needed to this Tx?
+* indirect => 1
* any_layout => number of segments
* default=> number of segments + 1
*/
- can_push = rte_mbuf_refcnt_read(txm) == 1 &&
- RTE_MBUF_DIRECT(txm) &&
- txm->nb_segs == 1 &&
- rte_pktmbuf_headroom(txm) >= hdr_size;
-
- slots = txm->nb_segs + !can_push;
+ slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
need = slots - vq->vq_free_cnt;
/* Positive value indicates it need free vring descriptors */
@@ -234,7 +241,8 @@ virtqueue_enqueue_single_packed_vec(struct virtnet_tx *txvq,
}
/* Enqueue Packet buffers */
- virtqueue_enqueue_xmit_packed(txvq, txm, slots, can_push, 1);
+ virtqueue_enqueue_xmit_packed(txvq, txm, slots, use_indirect,
+ can_push, 1);
txvq->stats.bytes += txm->pkt_len;
return 0;
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 7d910a0a1..753dfb85c 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -686,7 +686,8 @@ virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
static inline void
virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
- uint16_t needed, int can_push, int in_order)
+
[dpdk-dev] [PATCH 1/2] net/virtio: setup Tx region for packed ring
Add packed indirect descriptors format into virtio Tx region. When
initializing vring, packed indirect descriptors will be initialized if
ring type is packed.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 013a2904e..320f99836 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -609,10 +609,9 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t
vtpci_queue_idx)
txr = hdr_mz->addr;
memset(txr, 0, vq_size * sizeof(*txr));
for (i = 0; i < vq_size; i++) {
- struct vring_desc *start_dp = txr[i].tx_indir;
-
/* first indirect descriptor is always the tx header */
if (!vtpci_packed_queue(hw)) {
+ struct vring_desc *start_dp = txr[i].tx_indir;
vring_desc_init_split(start_dp,
RTE_DIM(txr[i].tx_indir));
start_dp->addr = txvq->virtio_net_hdr_mem
@@ -621,6 +620,16 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t
vtpci_queue_idx)
tx_hdr);
start_dp->len = hw->vtnet_hdr_size;
start_dp->flags = VRING_DESC_F_NEXT;
+ } else {
+ struct vring_packed_desc *start_dp =
+ txr[i].tx_packed_indir;
+ vring_desc_init_indirect_packed(start_dp,
+ RTE_DIM(txr[i].tx_packed_indir));
+ start_dp->addr = txvq->virtio_net_hdr_mem
+ + i * sizeof(*txr)
+ + offsetof(struct virtio_tx_region,
+ tx_hdr);
+ start_dp->len = hw->vtnet_hdr_size;
}
}
}
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 6ed50648c..7d910a0a1 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -329,8 +329,11 @@ struct virtio_net_hdr_mrg_rxbuf {
#define VIRTIO_MAX_TX_INDIRECT 8
struct virtio_tx_region {
struct virtio_net_hdr_mrg_rxbuf tx_hdr;
- struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT]
- __rte_aligned(16);
+ union {
+ struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT];
+ struct vring_packed_desc
+ tx_packed_indir[VIRTIO_MAX_TX_INDIRECT];
+ } __rte_aligned(16);
};
static inline int
@@ -368,6 +371,16 @@ vring_desc_init_split(struct vring_desc *dp, uint16_t n)
dp[i].next = VQ_RING_DESC_CHAIN_END;
}
+static inline void
+vring_desc_init_indirect_packed(struct vring_packed_desc *dp, int n)
+{
+ int i;
+ for (i = 0; i < n; i++) {
+ dp[i].id = (uint16_t)i;
+ dp[i].flags = VRING_DESC_F_WRITE;
+ }
+}
+
/**
* Tell the backend not to interrupt us. Implementation for packed virtqueues.
*/
--
2.17.1
[dpdk-dev] [PATCH v2] config: enable packet data prefetch
Data prefetch instruction can preload data into cpu’s hierarchical cache before data access. Virtualized data paths like virtio utilized this feature for acceleration. Since most modern cpus have support prefetch function, we can enable packet data prefetch as default. Signed-off-by: Marvin Liu --- v2: move define from meson.build to rte_config.h --- config/rte_config.h | 1 + 1 file changed, 1 insertion(+) diff --git a/config/rte_config.h b/config/rte_config.h index 0bae630fd9..8b007c4c31 100644 --- a/config/rte_config.h +++ b/config/rte_config.h @@ -101,6 +101,7 @@ #define RTE_LIBRTE_GRAPH_STATS 1 /** driver defines / +#define RTE_PMD_PACKET_PREFETCH 1 /* QuickAssist device */ /* Max. number of QuickAssist devices which can be attached */ -- 2.17.1
[dpdk-dev] [PATCH] build: enable packet data prefetch
Data prefetch instruction can preload data into cpu’s hierarchical
cache before data access. Virtualized data paths like virtio utilized
this feature for acceleration. Since most modern cpus have support
prefetch function, we can enable packet data prefetch as default.
Signed-off-by: Marvin Liu
diff --git a/config/meson.build b/config/meson.build
index 69f2aeb605..a0c828a437 100644
--- a/config/meson.build
+++ b/config/meson.build
@@ -109,6 +109,9 @@ if not is_windows
add_project_link_arguments('-Wl,--no-as-needed', language: 'c')
endif
+# do prefetch of packet data
+dpdk_conf.set('RTE_PMD_PACKET_PREFETCH', 1)
+
# use pthreads if available for the platform
if not is_windows
add_project_link_arguments('-pthread', language: 'c')
--
2.17.1
[dpdk-dev] [PATCH v2 4/5] vhost: add packed ring vectorized dequeue
Optimize vhost packed ring dequeue path with SIMD instructions. Four
descriptors status check and writeback are batched handled with AVX512
instructions. Address translation operations are also accelerated by
AVX512 instructions.
If platform or compiler not support vectorization, will fallback to
default path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/meson.build b/lib/librte_vhost/meson.build
index cc9aa65c67..c1481802d7 100644
--- a/lib/librte_vhost/meson.build
+++ b/lib/librte_vhost/meson.build
@@ -8,6 +8,22 @@ endif
if has_libnuma == 1
dpdk_conf.set10('RTE_LIBRTE_VHOST_NUMA', true)
endif
+
+if arch_subdir == 'x86'
+if not machine_args.contains('-mno-avx512f')
+if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+cflags += ['-DCC_AVX512_SUPPORT']
+vhost_avx512_lib = static_library('vhost_avx512_lib',
+ 'vhost_vec_avx.c',
+ dependencies: [static_rte_eal,
static_rte_mempool,
+ static_rte_mbuf,
static_rte_ethdev, static_rte_net],
+ include_directories: includes,
+ c_args: [cflags, '-mavx512f',
'-mavx512bw', '-mavx512vl'])
+objs +=
vhost_avx512_lib.extract_objects('vhost_vec_avx.c')
+endif
+endif
+endif
+
if (toolchain == 'gcc' and cc.version().version_compare('>=8.3.0'))
cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
elif (toolchain == 'clang' and cc.version().version_compare('>=3.7.0'))
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 4a81f18f01..fc7daf2145 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1124,4 +1124,12 @@ virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct
rte_mempool *mp,
return NULL;
}
+int
+vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
+struct vhost_virtqueue *vq,
+struct rte_mempool *mbuf_pool,
+struct rte_mbuf **pkts,
+uint16_t avail_idx,
+uintptr_t *desc_addrs,
+uint16_t *ids);
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_vec_avx.c b/lib/librte_vhost/vhost_vec_avx.c
new file mode 100644
index 00..dc5322d002
--- /dev/null
+++ b/lib/librte_vhost/vhost_vec_avx.c
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2016 Intel Corporation
+ */
+#include
+
+#include "vhost.h"
+
+#define BYTE_SIZE 8
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+#define DESC_FLAGS_SHORT_OFFSET (offsetof(struct vring_packed_desc, flags) / \
+ sizeof(uint16_t))
+
+#define DESC_FLAGS_SHORT_SIZE (sizeof(struct vring_packed_desc) / \
+ sizeof(uint16_t))
+#define BATCH_FLAGS_MASK (1 << DESC_FLAGS_SHORT_OFFSET | \
+ 1 << (DESC_FLAGS_SHORT_OFFSET + DESC_FLAGS_SHORT_SIZE) | \
+ 1 << (DESC_FLAGS_SHORT_OFFSET + DESC_FLAGS_SHORT_SIZE * 2) | \
+ 1 << (DESC_FLAGS_SHORT_OFFSET + DESC_FLAGS_SHORT_SIZE * 3))
+
+#define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+#define PACKED_FLAGS_MASK ((0ULL | VRING_DESC_F_AVAIL | VRING_DESC_F_USED) \
+ << FLAGS_BITS_OFFSET)
+#define PACKED_AVAIL_FLAG ((0ULL | VRING_DESC_F_AVAIL) << FLAGS_BITS_OFFSET)
+#define PACKED_AVAIL_FLAG_WRAP ((0ULL | VRING_DESC_F_USED) << \
+ FLAGS_BITS_OFFSET)
+
+#define DESC_FLAGS_POS 0xaa
+#define MBUF_LENS_POS 0x
+
+int
+vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
+struct vhost_virtqueue *vq,
+struct rte_mempool *mbuf_pool,
+struct rte_mbuf **pkts,
+uint16_t avail_idx,
+uintptr_t *desc_addrs,
+uint16_t *ids)
+{
+ struct vring_packed_desc *descs = vq->desc_packed;
+ uint32_t descs_st
[dpdk-dev] [PATCH v2 5/5] vhost: add packed ring vectorized enqueue
Optimize vhost packed ring enqueue path with SIMD instructions. Four
descriptors status and length are batched handled with AVX512
instructions. Address translation operations are also accelerated
by AVX512 instructions.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index fc7daf2145..b78b2c5c1b 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1132,4 +1132,10 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
uint16_t avail_idx,
uintptr_t *desc_addrs,
uint16_t *ids);
+
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts);
+
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_vec_avx.c b/lib/librte_vhost/vhost_vec_avx.c
index dc5322d002..7d2250ed86 100644
--- a/lib/librte_vhost/vhost_vec_avx.c
+++ b/lib/librte_vhost/vhost_vec_avx.c
@@ -35,9 +35,15 @@
#define PACKED_AVAIL_FLAG ((0ULL | VRING_DESC_F_AVAIL) << FLAGS_BITS_OFFSET)
#define PACKED_AVAIL_FLAG_WRAP ((0ULL | VRING_DESC_F_USED) << \
FLAGS_BITS_OFFSET)
+#define PACKED_WRITE_AVAIL_FLAG (PACKED_AVAIL_FLAG | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
+#define PACKED_WRITE_AVAIL_FLAG_WRAP (PACKED_AVAIL_FLAG_WRAP | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
#define DESC_FLAGS_POS 0xaa
#define MBUF_LENS_POS 0x
+#define DESC_LENS_POS 0x
+#define DESC_LENS_FLAGS_POS 0xB0B0B0B0
int
vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
@@ -179,3 +185,154 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
return -1;
}
+
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts)
+{
+ struct vring_packed_desc *descs = vq->desc_packed;
+ uint16_t avail_idx = vq->last_avail_idx;
+ uint64_t desc_addrs[PACKED_BATCH_SIZE];
+ uint32_t buf_offset = dev->vhost_hlen;
+ uint32_t desc_status;
+ uint64_t lens[PACKED_BATCH_SIZE];
+ uint16_t i;
+ void *desc_addr;
+ uint8_t cmp_low, cmp_high, cmp_result;
+
+ if (unlikely(avail_idx & PACKED_BATCH_MASK))
+ return -1;
+
+ /* check refcnt and nb_segs */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+
+ /* load four mbufs rearm data */
+ __m256i mbufs = _mm256_set_epi64x(
+ *pkts[3]->rearm_data,
+ *pkts[2]->rearm_data,
+ *pkts[1]->rearm_data,
+ *pkts[0]->rearm_data);
+
+ uint16_t cmp = _mm256_cmpneq_epu16_mask(mbufs, mbuf_ref);
+ if (cmp & MBUF_LENS_POS)
+ return -1;
+
+ /* check desc status */
+ desc_addr = &vq->desc_packed[avail_idx];
+ __m512i desc_vec = _mm512_loadu_si512(desc_addr);
+
+ __m512i avail_flag_vec;
+ __m512i used_flag_vec;
+ if (vq->avail_wrap_counter) {
+#if defined(RTE_ARCH_I686)
+ avail_flag_vec = _mm512_set4_epi64(PACKED_WRITE_AVAIL_FLAG,
+ 0x0, PACKED_WRITE_AVAIL_FLAG, 0x0);
+ used_flag_vec = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
+ PACKED_FLAGS_MASK, 0x0);
+#else
+ avail_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_WRITE_AVAIL_FLAG);
+ used_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_FLAGS_MASK);
+#endif
+ } else {
+#if defined(RTE_ARCH_I686)
+ avail_flag_vec = _mm512_set4_epi64(
+ PACKED_WRITE_AVAIL_FLAG_WRAP, 0x0,
+ PACKED_WRITE_AVAIL_FLAG, 0x0);
+ used_flag_vec = _mm512_set4_epi64(0x0, 0x0, 0x0, 0x0);
+#else
+ avail_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_WRITE_AVAIL_FLAG_WRAP);
+ used_flag_vec = _mm512_setzero_epi32();
+#endif
+ }
+
+ desc_status = _mm512_mask_cmp_epu16_mask(BATCH_FLAGS_MASK, desc_vec,
+ avail_flag_vec, _MM_CMPINT_NE);
+ if (desc_status)
+ return -1;
+
+ if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) {
+ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ uint64_t size = (uint64_t)descs[avail_idx + i].len;
+ desc_addrs[i] = __vhost_iova_to_vva(dev, vq,
+ descs[avail_idx + i].addr, &size,
+
[dpdk-dev] [PATCH v2 3/5] vhost: prepare memory regions addresses
Prepare memory regions guest physical addresses for vectorized data
path. These information will be utilized by SIMD instructions to find
matched region index.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 5a5c945551..4a81f18f01 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -52,6 +52,8 @@
#define ASYNC_MAX_POLL_SEG 255
+#define MAX_NREGIONS 8
+
#define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2)
#define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2)
@@ -375,6 +377,8 @@ struct inflight_mem_info {
struct virtio_net {
/* Frontend (QEMU) memory and memory region information */
struct rte_vhost_memory *mem;
+ uint64_tregions_low_addrs[MAX_NREGIONS];
+ uint64_tregions_high_addrs[MAX_NREGIONS];
uint64_tfeatures;
uint64_tprotocol_features;
int vid;
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index c3c924faec..89e75e9e71 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -1291,6 +1291,17 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
}
}
+ RTE_BUILD_BUG_ON(VHOST_MEMORY_MAX_NREGIONS != 8);
+ if (dev->vectorized) {
+ for (i = 0; i < memory->nregions; i++) {
+ dev->regions_low_addrs[i] =
+ memory->regions[i].guest_phys_addr;
+ dev->regions_high_addrs[i] =
+ memory->regions[i].guest_phys_addr +
+ memory->regions[i].memory_size;
+ }
+ }
+
for (i = 0; i < dev->nr_vring; i++) {
struct vhost_virtqueue *vq = dev->virtqueue[i];
--
2.17.1
[dpdk-dev] [PATCH v2 0/5] vhost add vectorized data path
Packed ring format is imported since virtio spec 1.1. All descriptors are compacted into one single ring when packed ring format is on. It is straight forward that ring operations can be accelerated by utilizing SIMD instructions. This patch set will introduce vectorized data path in vhost library. If vectorized option is on, operations like descs check, descs writeback, address translation will be accelerated by SIMD instructions. Vhost application can choose whether using vectorized acceleration, it is like external buffer and zero copy features. If platform or ring format not support vectorized function, vhost will fallback to use default batch function. There will be no impact in current data path. v2: * add vIOMMU support * add dequeue offloading * rebase code Marvin Liu (5): vhost: add vectorized data path vhost: reuse packed ring functions vhost: prepare memory regions addresses vhost: add packed ring vectorized dequeue vhost: add packed ring vectorized enqueue doc/guides/nics/vhost.rst | 5 + doc/guides/prog_guide/vhost_lib.rst | 12 + drivers/net/vhost/rte_eth_vhost.c | 17 +- lib/librte_vhost/meson.build| 16 ++ lib/librte_vhost/rte_vhost.h| 1 + lib/librte_vhost/socket.c | 5 + lib/librte_vhost/vhost.c| 11 + lib/librte_vhost/vhost.h| 235 +++ lib/librte_vhost/vhost_user.c | 11 + lib/librte_vhost/vhost_vec_avx.c| 338 lib/librte_vhost/virtio_net.c | 257 - 11 files changed, 692 insertions(+), 216 deletions(-) create mode 100644 lib/librte_vhost/vhost_vec_avx.c -- 2.17.1
[dpdk-dev] [PATCH v2 2/5] vhost: reuse packed ring functions
Move parse_ethernet, offload, extbuf functions to header file. These
functions will be reused by vhost vectorized path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index b556eb3bf6..5a5c945551 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -20,6 +20,10 @@
#include
#include
+#include
+#include
+#include
+#include
#include "rte_vhost.h"
#include "rte_vdpa.h"
#include "rte_vdpa_dev.h"
@@ -905,4 +909,215 @@ put_zmbuf(struct zcopy_mbuf *zmbuf)
zmbuf->in_use = 0;
}
+static __rte_always_inline bool
+virtio_net_is_inorder(struct virtio_net *dev)
+{
+ return dev->features & (1ULL << VIRTIO_F_IN_ORDER);
+}
+
+static __rte_always_inline void
+parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ void *l3_hdr = NULL;
+ struct rte_ether_hdr *eth_hdr;
+ uint16_t ethertype;
+
+ eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
+
+ m->l2_len = sizeof(struct rte_ether_hdr);
+ ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
+
+ if (ethertype == RTE_ETHER_TYPE_VLAN) {
+ struct rte_vlan_hdr *vlan_hdr =
+ (struct rte_vlan_hdr *)(eth_hdr + 1);
+
+ m->l2_len += sizeof(struct rte_vlan_hdr);
+ ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
+ }
+
+ l3_hdr = (char *)eth_hdr + m->l2_len;
+
+ switch (ethertype) {
+ case RTE_ETHER_TYPE_IPV4:
+ ipv4_hdr = l3_hdr;
+ *l4_proto = ipv4_hdr->next_proto_id;
+ m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV4;
+ break;
+ case RTE_ETHER_TYPE_IPV6:
+ ipv6_hdr = l3_hdr;
+ *l4_proto = ipv6_hdr->proto;
+ m->l3_len = sizeof(struct rte_ipv6_hdr);
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV6;
+ break;
+ default:
+ m->l3_len = 0;
+ *l4_proto = 0;
+ *l4_hdr = NULL;
+ break;
+ }
+}
+
+static __rte_always_inline bool
+virtio_net_with_host_offload(struct virtio_net *dev)
+{
+ if (dev->features &
+ ((1ULL << VIRTIO_NET_F_CSUM) |
+(1ULL << VIRTIO_NET_F_HOST_ECN) |
+(1ULL << VIRTIO_NET_F_HOST_TSO4) |
+(1ULL << VIRTIO_NET_F_HOST_TSO6) |
+(1ULL << VIRTIO_NET_F_HOST_UFO)))
+ return true;
+
+ return false;
+}
+
+static __rte_always_inline void
+vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
+{
+ uint16_t l4_proto = 0;
+ void *l4_hdr = NULL;
+ struct rte_tcp_hdr *tcp_hdr = NULL;
+
+ if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
+ return;
+
+ parse_ethernet(m, &l4_proto, &l4_hdr);
+ if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+ if (hdr->csum_start == (m->l2_len + m->l3_len)) {
+ switch (hdr->csum_offset) {
+ case (offsetof(struct rte_tcp_hdr, cksum)):
+ if (l4_proto == IPPROTO_TCP)
+ m->ol_flags |= PKT_TX_TCP_CKSUM;
+ break;
+ case (offsetof(struct rte_udp_hdr, dgram_cksum)):
+ if (l4_proto == IPPROTO_UDP)
+ m->ol_flags |= PKT_TX_UDP_CKSUM;
+ break;
+ case (offsetof(struct rte_sctp_hdr, cksum)):
+ if (l4_proto == IPPROTO_SCTP)
+ m->ol_flags |= PKT_TX_SCTP_CKSUM;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+ switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+ case VIRTIO_NET_HDR_GSO_TCPV4:
+ case VIRTIO_NET_HDR_GSO_TCPV6:
+ tcp_hdr = l4_hdr;
+ m->ol_flags |= PKT_TX_TCP_SEG;
+ m->tso_segsz = hdr->gso_size;
+ m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
+ break;
+ case VIRTIO_NET_HDR_GSO_UDP:
+ m->ol_flags |= PKT_TX_UDP_S
[dpdk-dev] [PATCH v2 1/5] vhost: add vectorized data path
Packed ring operations are split into batch and single functions for
performance perspective. Ring operations in batch function can be
accelerated by SIMD instructions like AVX512.
So introduce vectorized parameter in vhost. Vectorized data path can be
selected if platform and ring format matched requirements. Otherwise
will fallback to original data path.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/vhost.rst b/doc/guides/nics/vhost.rst
index d36f3120b2..efdaf4de09 100644
--- a/doc/guides/nics/vhost.rst
+++ b/doc/guides/nics/vhost.rst
@@ -64,6 +64,11 @@ The user can specify below arguments in `--vdev` option.
It is used to enable external buffer support in vhost library.
(Default: 0 (disabled))
+#. ``vectorized``:
+
+It is used to enable vectorized data path support in vhost library.
+(Default: 0 (disabled))
+
Vhost PMD event handling
diff --git a/doc/guides/prog_guide/vhost_lib.rst
b/doc/guides/prog_guide/vhost_lib.rst
index b892eec67a..d5d421441c 100644
--- a/doc/guides/prog_guide/vhost_lib.rst
+++ b/doc/guides/prog_guide/vhost_lib.rst
@@ -162,6 +162,18 @@ The following is an overview of some key Vhost API
functions:
It is disabled by default.
+ - ``RTE_VHOST_USER_VECTORIZED``
+Vectorized data path will used when this flag is set. When packed ring
+enabled, available descriptors are stored from frontend driver in sequence.
+SIMD instructions like AVX can be used to handle multiple descriptors
+simultaneously. Thus can accelerate the throughput of ring operations.
+
+* Only packed ring has vectorized data path.
+
+* Will fallback to normal datapath if no vectorization support.
+
+It is disabled by default.
+
* ``rte_vhost_driver_set_features(path, features)``
This function sets the feature bits the vhost-user driver supports. The
diff --git a/drivers/net/vhost/rte_eth_vhost.c
b/drivers/net/vhost/rte_eth_vhost.c
index e55278af69..2ba5a2a076 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -35,6 +35,7 @@ enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
#define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso"
#define ETH_VHOST_LINEAR_BUF "linear-buffer"
#define ETH_VHOST_EXT_BUF "ext-buffer"
+#define ETH_VHOST_VECTORIZED "vectorized"
#define VHOST_MAX_PKT_BURST 32
static const char *valid_arguments[] = {
@@ -47,6 +48,7 @@ static const char *valid_arguments[] = {
ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
ETH_VHOST_LINEAR_BUF,
ETH_VHOST_EXT_BUF,
+ ETH_VHOST_VECTORIZED,
NULL
};
@@ -1507,6 +1509,7 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
int tso = 0;
int linear_buf = 0;
int ext_buf = 0;
+ int vectorized = 0;
struct rte_eth_dev *eth_dev;
const char *name = rte_vdev_device_name(dev);
@@ -1626,6 +1629,17 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
flags |= RTE_VHOST_USER_EXTBUF_SUPPORT;
}
+ if (rte_kvargs_count(kvlist, ETH_VHOST_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ ETH_VHOST_VECTORIZED,
+ &open_int, &vectorized);
+ if (ret < 0)
+ goto out_free;
+
+ if (vectorized == 1)
+ flags |= RTE_VHOST_USER_VECTORIZED;
+ }
+
if (dev->device.numa_node == SOCKET_ID_ANY)
dev->device.numa_node = rte_socket_id();
@@ -1679,4 +1693,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_vhost,
"postcopy-support=<0|1> "
"tso=<0|1> "
"linear-buffer=<0|1> "
- "ext-buffer=<0|1>");
+ "ext-buffer=<0|1> "
+ "vectorized=<0|1>");
diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
index a94c84134d..c7f946c6c1 100644
--- a/lib/librte_vhost/rte_vhost.h
+++ b/lib/librte_vhost/rte_vhost.h
@@ -36,6 +36,7 @@ extern "C" {
/* support only linear buffers (no chained mbufs) */
#define RTE_VHOST_USER_LINEARBUF_SUPPORT (1ULL << 6)
#define RTE_VHOST_USER_ASYNC_COPY (1ULL << 7)
+#define RTE_VHOST_USER_VECTORIZED (1ULL << 8)
/* Features. */
#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 73e1dca95e..cc11244693 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -43,6 +43,7 @@ struct vhost_user_socket {
bool extbuf;
bool linearbuf;
bool async_copy;
+ bool vectorized;
/*
* The "supported_features" indicates the feature bits the
@@ -245,6 +246,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket
*vsocket)
dev->async_copy = 1;
}
+ if (vsocket->
[dpdk-dev] [PATCH v1 5/5] vhost: add packed ring vectorized enqueue
Optimize vhost packed ring enqueue path with SIMD instructions. Four
descriptors status and length are batched handled with AVX512
instructions. Address translation operations are also accelerated
by AVX512 instructions.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index fc7daf2145..b78b2c5c1b 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1132,4 +1132,10 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
uint16_t avail_idx,
uintptr_t *desc_addrs,
uint16_t *ids);
+
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts);
+
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_vec_avx.c b/lib/librte_vhost/vhost_vec_avx.c
index e8361d18fa..12b902253a 100644
--- a/lib/librte_vhost/vhost_vec_avx.c
+++ b/lib/librte_vhost/vhost_vec_avx.c
@@ -35,9 +35,15 @@
#define PACKED_AVAIL_FLAG ((0ULL | VRING_DESC_F_AVAIL) << FLAGS_BITS_OFFSET)
#define PACKED_AVAIL_FLAG_WRAP ((0ULL | VRING_DESC_F_USED) << \
FLAGS_BITS_OFFSET)
+#define PACKED_WRITE_AVAIL_FLAG (PACKED_AVAIL_FLAG | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
+#define PACKED_WRITE_AVAIL_FLAG_WRAP (PACKED_AVAIL_FLAG_WRAP | \
+ ((0ULL | VRING_DESC_F_WRITE) << FLAGS_BITS_OFFSET))
#define DESC_FLAGS_POS 0xaa
#define MBUF_LENS_POS 0x
+#define DESC_LENS_POS 0x
+#define DESC_LENS_FLAGS_POS 0xB0B0B0B0
int
vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
@@ -150,3 +156,137 @@ vhost_reserve_avail_batch_packed_avx(struct virtio_net
*dev,
return -1;
}
+
+int
+virtio_dev_rx_batch_packed_avx(struct virtio_net *dev,
+ struct vhost_virtqueue *vq,
+ struct rte_mbuf **pkts)
+{
+ struct vring_packed_desc *descs = vq->desc_packed;
+ uint16_t avail_idx = vq->last_avail_idx;
+ uint64_t desc_addrs[PACKED_BATCH_SIZE];
+ uint32_t buf_offset = dev->vhost_hlen;
+ uint32_t desc_status;
+ uint64_t lens[PACKED_BATCH_SIZE];
+ uint16_t i;
+ void *desc_addr;
+ uint8_t cmp_low, cmp_high, cmp_result;
+
+ if (unlikely(avail_idx & PACKED_BATCH_MASK))
+ return -1;
+
+ /* check refcnt and nb_segs */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+
+ /* load four mbufs rearm data */
+ __m256i mbufs = _mm256_set_epi64x(
+ *pkts[3]->rearm_data,
+ *pkts[2]->rearm_data,
+ *pkts[1]->rearm_data,
+ *pkts[0]->rearm_data);
+
+ uint16_t cmp = _mm256_cmpneq_epu16_mask(mbufs, mbuf_ref);
+ if (cmp & MBUF_LENS_POS)
+ return -1;
+
+ /* check desc status */
+ desc_addr = &vq->desc_packed[avail_idx];
+ __m512i desc_vec = _mm512_loadu_si512(desc_addr);
+
+ __m512i avail_flag_vec;
+ __m512i used_flag_vec;
+ if (vq->avail_wrap_counter) {
+#if defined(RTE_ARCH_I686)
+ avail_flag_vec = _mm512_set4_epi64(PACKED_WRITE_AVAIL_FLAG,
+ 0x0, PACKED_WRITE_AVAIL_FLAG, 0x0);
+ used_flag_vec = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
+ PACKED_FLAGS_MASK, 0x0);
+#else
+ avail_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_WRITE_AVAIL_FLAG);
+ used_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_FLAGS_MASK);
+#endif
+ } else {
+#if defined(RTE_ARCH_I686)
+ avail_flag_vec = _mm512_set4_epi64(
+ PACKED_WRITE_AVAIL_FLAG_WRAP, 0x0,
+ PACKED_WRITE_AVAIL_FLAG, 0x0);
+ used_flag_vec = _mm512_set4_epi64(0x0, 0x0, 0x0, 0x0);
+#else
+ avail_flag_vec = _mm512_maskz_set1_epi64(DESC_FLAGS_POS,
+ PACKED_WRITE_AVAIL_FLAG_WRAP);
+ used_flag_vec = _mm512_setzero_epi32();
+#endif
+ }
+
+ desc_status = _mm512_mask_cmp_epu16_mask(BATCH_FLAGS_MASK, desc_vec,
+ avail_flag_vec, _MM_CMPINT_NE);
+ if (desc_status)
+ return -1;
+
+ /* check buffer fit into one region & translate address */
+ __m512i regions_low_addrs =
+ _mm512_loadu_si512((void *)&dev->regions_low_addrs);
+ __m512i regions_high_addrs =
+ _mm512_loadu_si512((void *)&dev->regions_high_addrs);
+ vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+
[dpdk-dev] [PATCH v1 4/5] vhost: add packed ring vectorized dequeue
Optimize vhost packed ring dequeue path with SIMD instructions. Four
descriptors status check and writeback are batched handled with AVX512
instructions. Address translation operations are also accelerated by
AVX512 instructions.
If platform or compiler not support vectorization, will fallback to
default path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/Makefile b/lib/librte_vhost/Makefile
index 4f2f3e47da..c0cd7d498f 100644
--- a/lib/librte_vhost/Makefile
+++ b/lib/librte_vhost/Makefile
@@ -31,6 +31,13 @@ CFLAGS += -DVHOST_ICC_UNROLL_PRAGMA
endif
endif
+ifneq ($(FORCE_DISABLE_AVX512), y)
+CC_AVX512_SUPPORT=\
+$(shell $(CC) -march=native -dM -E - &1 | \
+sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+grep -q AVX512 && echo 1)
+endif
+
ifeq ($(CONFIG_RTE_LIBRTE_VHOST_NUMA),y)
LDLIBS += -lnuma
endif
@@ -40,6 +47,12 @@ LDLIBS += -lrte_eal -lrte_mempool -lrte_mbuf -lrte_ethdev
-lrte_net
SRCS-$(CONFIG_RTE_LIBRTE_VHOST) := fd_man.c iotlb.c socket.c vhost.c \
vhost_user.c virtio_net.c vdpa.c
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost_vec_avx.c
+CFLAGS_vhost_vec_avx.o += -mavx512f -mavx512bw -mavx512vl
+endif
+
# install includes
SYMLINK-$(CONFIG_RTE_LIBRTE_VHOST)-include += rte_vhost.h rte_vdpa.h \
rte_vdpa_dev.h rte_vhost_async.h
diff --git a/lib/librte_vhost/meson.build b/lib/librte_vhost/meson.build
index cc9aa65c67..c1481802d7 100644
--- a/lib/librte_vhost/meson.build
+++ b/lib/librte_vhost/meson.build
@@ -8,6 +8,22 @@ endif
if has_libnuma == 1
dpdk_conf.set10('RTE_LIBRTE_VHOST_NUMA', true)
endif
+
+if arch_subdir == 'x86'
+if not machine_args.contains('-mno-avx512f')
+if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+cflags += ['-DCC_AVX512_SUPPORT']
+vhost_avx512_lib = static_library('vhost_avx512_lib',
+ 'vhost_vec_avx.c',
+ dependencies: [static_rte_eal,
static_rte_mempool,
+ static_rte_mbuf,
static_rte_ethdev, static_rte_net],
+ include_directories: includes,
+ c_args: [cflags, '-mavx512f',
'-mavx512bw', '-mavx512vl'])
+objs +=
vhost_avx512_lib.extract_objects('vhost_vec_avx.c')
+endif
+endif
+endif
+
if (toolchain == 'gcc' and cc.version().version_compare('>=8.3.0'))
cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
elif (toolchain == 'clang' and cc.version().version_compare('>=3.7.0'))
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 4a81f18f01..fc7daf2145 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -1124,4 +1124,12 @@ virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct
rte_mempool *mp,
return NULL;
}
+int
+vhost_reserve_avail_batch_packed_avx(struct virtio_net *dev,
+struct vhost_virtqueue *vq,
+struct rte_mempool *mbuf_pool,
+struct rte_mbuf **pkts,
+uint16_t avail_idx,
+uintptr_t *desc_addrs,
+uint16_t *ids);
#endif /* _VHOST_NET_CDEV_H_ */
diff --git a/lib/librte_vhost/vhost_vec_avx.c b/lib/librte_vhost/vhost_vec_avx.c
new file mode 100644
index 00..e8361d18fa
--- /dev/null
+++ b/lib/librte_vhost/vhost_vec_avx.c
@@ -0,0 +1,152 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2016 Intel Corporation
+ */
+#include
+
+#include "vhost.h"
+
+#define BYTE_SIZE 8
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+#define DESC_FLAGS_SHORT_OFFSET (offsetof(struct vring_packed_desc, flags) / \
+ sizeof(uint16_t))
+
+#define DESC_FLAGS_SHORT_SIZE (sizeof(struct vring_packed_desc) / \
+ sizeof(uint16_t))
+#define BATCH_FLAGS_MASK (1 << DESC_FLAGS_SHORT_OFFSET | \
+ 1 << (DESC_FLAGS_SHORT_OFFSET + DESC_FLAGS_SHORT_S
[dpdk-dev] [PATCH v1 0/5] vhost add vectorized data path
Packed ring format is imported since virtio spec 1.1. All descriptors are compacted into one single ring when packed ring format is on. It is straight forward that ring operations can be accelerated by utilizing SIMD instructions. This patch set will introduce vectorized data path in vhost library. If vectorized option is on, operations like descs check, descs writeback, address translation will be accelerated by SIMD instructions. Vhost application can choose whether using vectorized acceleration, it is like external buffer and zero copy features. If platform or ring format not support vectorized function, vhost will fallback to use default batch function. There will be no impact in current data path. Marvin Liu (5): vhost: add vectorized data path vhost: reuse packed ring functions vhost: prepare memory regions addresses vhost: add packed ring vectorized dequeue vhost: add packed ring vectorized enqueue doc/guides/nics/vhost.rst | 5 + doc/guides/prog_guide/vhost_lib.rst | 12 ++ drivers/net/vhost/rte_eth_vhost.c | 17 +- lib/librte_vhost/Makefile | 13 ++ lib/librte_vhost/meson.build| 16 ++ lib/librte_vhost/rte_vhost.h| 1 + lib/librte_vhost/socket.c | 5 + lib/librte_vhost/vhost.c| 11 ++ lib/librte_vhost/vhost.h| 235 ++ lib/librte_vhost/vhost_user.c | 11 ++ lib/librte_vhost/vhost_vec_avx.c| 292 lib/librte_vhost/virtio_net.c | 257 12 files changed, 659 insertions(+), 216 deletions(-) create mode 100644 lib/librte_vhost/vhost_vec_avx.c -- 2.17.1
[dpdk-dev] [PATCH v1 3/5] vhost: prepare memory regions addresses
Prepare memory regions guest physical addresses for vectorized data
path. These information will be utilized by SIMD instructions to find
matched region index.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 5a5c945551..4a81f18f01 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -52,6 +52,8 @@
#define ASYNC_MAX_POLL_SEG 255
+#define MAX_NREGIONS 8
+
#define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2)
#define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2)
@@ -375,6 +377,8 @@ struct inflight_mem_info {
struct virtio_net {
/* Frontend (QEMU) memory and memory region information */
struct rte_vhost_memory *mem;
+ uint64_tregions_low_addrs[MAX_NREGIONS];
+ uint64_tregions_high_addrs[MAX_NREGIONS];
uint64_tfeatures;
uint64_tprotocol_features;
int vid;
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index c3c924faec..89e75e9e71 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -1291,6 +1291,17 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
}
}
+ RTE_BUILD_BUG_ON(VHOST_MEMORY_MAX_NREGIONS != 8);
+ if (dev->vectorized) {
+ for (i = 0; i < memory->nregions; i++) {
+ dev->regions_low_addrs[i] =
+ memory->regions[i].guest_phys_addr;
+ dev->regions_high_addrs[i] =
+ memory->regions[i].guest_phys_addr +
+ memory->regions[i].memory_size;
+ }
+ }
+
for (i = 0; i < dev->nr_vring; i++) {
struct vhost_virtqueue *vq = dev->virtqueue[i];
--
2.17.1
[dpdk-dev] [PATCH v1 1/5] vhost: add vectorized data path
Packed ring operations are split into batch and single functions for
performance perspective. Ring operations in batch function can be
accelerated by SIMD instructions like AVX512.
So introduce vectorized parameter in vhost. Vectorized data path can be
selected if platform and ring format matched requirements. Otherwise
will fallback to original data path.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/vhost.rst b/doc/guides/nics/vhost.rst
index d36f3120b2..efdaf4de09 100644
--- a/doc/guides/nics/vhost.rst
+++ b/doc/guides/nics/vhost.rst
@@ -64,6 +64,11 @@ The user can specify below arguments in `--vdev` option.
It is used to enable external buffer support in vhost library.
(Default: 0 (disabled))
+#. ``vectorized``:
+
+It is used to enable vectorized data path support in vhost library.
+(Default: 0 (disabled))
+
Vhost PMD event handling
diff --git a/doc/guides/prog_guide/vhost_lib.rst
b/doc/guides/prog_guide/vhost_lib.rst
index b892eec67a..d5d421441c 100644
--- a/doc/guides/prog_guide/vhost_lib.rst
+++ b/doc/guides/prog_guide/vhost_lib.rst
@@ -162,6 +162,18 @@ The following is an overview of some key Vhost API
functions:
It is disabled by default.
+ - ``RTE_VHOST_USER_VECTORIZED``
+Vectorized data path will used when this flag is set. When packed ring
+enabled, available descriptors are stored from frontend driver in sequence.
+SIMD instructions like AVX can be used to handle multiple descriptors
+simultaneously. Thus can accelerate the throughput of ring operations.
+
+* Only packed ring has vectorized data path.
+
+* Will fallback to normal datapath if no vectorization support.
+
+It is disabled by default.
+
* ``rte_vhost_driver_set_features(path, features)``
This function sets the feature bits the vhost-user driver supports. The
diff --git a/drivers/net/vhost/rte_eth_vhost.c
b/drivers/net/vhost/rte_eth_vhost.c
index e55278af69..2ba5a2a076 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -35,6 +35,7 @@ enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM};
#define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso"
#define ETH_VHOST_LINEAR_BUF "linear-buffer"
#define ETH_VHOST_EXT_BUF "ext-buffer"
+#define ETH_VHOST_VECTORIZED "vectorized"
#define VHOST_MAX_PKT_BURST 32
static const char *valid_arguments[] = {
@@ -47,6 +48,7 @@ static const char *valid_arguments[] = {
ETH_VHOST_VIRTIO_NET_F_HOST_TSO,
ETH_VHOST_LINEAR_BUF,
ETH_VHOST_EXT_BUF,
+ ETH_VHOST_VECTORIZED,
NULL
};
@@ -1507,6 +1509,7 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
int tso = 0;
int linear_buf = 0;
int ext_buf = 0;
+ int vectorized = 0;
struct rte_eth_dev *eth_dev;
const char *name = rte_vdev_device_name(dev);
@@ -1626,6 +1629,17 @@ rte_pmd_vhost_probe(struct rte_vdev_device *dev)
flags |= RTE_VHOST_USER_EXTBUF_SUPPORT;
}
+ if (rte_kvargs_count(kvlist, ETH_VHOST_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ ETH_VHOST_VECTORIZED,
+ &open_int, &vectorized);
+ if (ret < 0)
+ goto out_free;
+
+ if (vectorized == 1)
+ flags |= RTE_VHOST_USER_VECTORIZED;
+ }
+
if (dev->device.numa_node == SOCKET_ID_ANY)
dev->device.numa_node = rte_socket_id();
@@ -1679,4 +1693,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_vhost,
"postcopy-support=<0|1> "
"tso=<0|1> "
"linear-buffer=<0|1> "
- "ext-buffer=<0|1>");
+ "ext-buffer=<0|1> "
+ "vectorized=<0|1>");
diff --git a/lib/librte_vhost/rte_vhost.h b/lib/librte_vhost/rte_vhost.h
index a94c84134d..c7f946c6c1 100644
--- a/lib/librte_vhost/rte_vhost.h
+++ b/lib/librte_vhost/rte_vhost.h
@@ -36,6 +36,7 @@ extern "C" {
/* support only linear buffers (no chained mbufs) */
#define RTE_VHOST_USER_LINEARBUF_SUPPORT (1ULL << 6)
#define RTE_VHOST_USER_ASYNC_COPY (1ULL << 7)
+#define RTE_VHOST_USER_VECTORIZED (1ULL << 8)
/* Features. */
#ifndef VIRTIO_NET_F_GUEST_ANNOUNCE
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 73e1dca95e..cc11244693 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -43,6 +43,7 @@ struct vhost_user_socket {
bool extbuf;
bool linearbuf;
bool async_copy;
+ bool vectorized;
/*
* The "supported_features" indicates the feature bits the
@@ -245,6 +246,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket
*vsocket)
dev->async_copy = 1;
}
+ if (vsocket->
[dpdk-dev] [PATCH v1 2/5] vhost: reuse packed ring functions
Move parse_ethernet, offload, extbuf functions to header file. These
functions will be reused by vhost vectorized path.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index b556eb3bf6..5a5c945551 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -20,6 +20,10 @@
#include
#include
+#include
+#include
+#include
+#include
#include "rte_vhost.h"
#include "rte_vdpa.h"
#include "rte_vdpa_dev.h"
@@ -905,4 +909,215 @@ put_zmbuf(struct zcopy_mbuf *zmbuf)
zmbuf->in_use = 0;
}
+static __rte_always_inline bool
+virtio_net_is_inorder(struct virtio_net *dev)
+{
+ return dev->features & (1ULL << VIRTIO_F_IN_ORDER);
+}
+
+static __rte_always_inline void
+parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr)
+{
+ struct rte_ipv4_hdr *ipv4_hdr;
+ struct rte_ipv6_hdr *ipv6_hdr;
+ void *l3_hdr = NULL;
+ struct rte_ether_hdr *eth_hdr;
+ uint16_t ethertype;
+
+ eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
+
+ m->l2_len = sizeof(struct rte_ether_hdr);
+ ethertype = rte_be_to_cpu_16(eth_hdr->ether_type);
+
+ if (ethertype == RTE_ETHER_TYPE_VLAN) {
+ struct rte_vlan_hdr *vlan_hdr =
+ (struct rte_vlan_hdr *)(eth_hdr + 1);
+
+ m->l2_len += sizeof(struct rte_vlan_hdr);
+ ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto);
+ }
+
+ l3_hdr = (char *)eth_hdr + m->l2_len;
+
+ switch (ethertype) {
+ case RTE_ETHER_TYPE_IPV4:
+ ipv4_hdr = l3_hdr;
+ *l4_proto = ipv4_hdr->next_proto_id;
+ m->l3_len = (ipv4_hdr->version_ihl & 0x0f) * 4;
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV4;
+ break;
+ case RTE_ETHER_TYPE_IPV6:
+ ipv6_hdr = l3_hdr;
+ *l4_proto = ipv6_hdr->proto;
+ m->l3_len = sizeof(struct rte_ipv6_hdr);
+ *l4_hdr = (char *)l3_hdr + m->l3_len;
+ m->ol_flags |= PKT_TX_IPV6;
+ break;
+ default:
+ m->l3_len = 0;
+ *l4_proto = 0;
+ *l4_hdr = NULL;
+ break;
+ }
+}
+
+static __rte_always_inline bool
+virtio_net_with_host_offload(struct virtio_net *dev)
+{
+ if (dev->features &
+ ((1ULL << VIRTIO_NET_F_CSUM) |
+(1ULL << VIRTIO_NET_F_HOST_ECN) |
+(1ULL << VIRTIO_NET_F_HOST_TSO4) |
+(1ULL << VIRTIO_NET_F_HOST_TSO6) |
+(1ULL << VIRTIO_NET_F_HOST_UFO)))
+ return true;
+
+ return false;
+}
+
+static __rte_always_inline void
+vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m)
+{
+ uint16_t l4_proto = 0;
+ void *l4_hdr = NULL;
+ struct rte_tcp_hdr *tcp_hdr = NULL;
+
+ if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
+ return;
+
+ parse_ethernet(m, &l4_proto, &l4_hdr);
+ if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+ if (hdr->csum_start == (m->l2_len + m->l3_len)) {
+ switch (hdr->csum_offset) {
+ case (offsetof(struct rte_tcp_hdr, cksum)):
+ if (l4_proto == IPPROTO_TCP)
+ m->ol_flags |= PKT_TX_TCP_CKSUM;
+ break;
+ case (offsetof(struct rte_udp_hdr, dgram_cksum)):
+ if (l4_proto == IPPROTO_UDP)
+ m->ol_flags |= PKT_TX_UDP_CKSUM;
+ break;
+ case (offsetof(struct rte_sctp_hdr, cksum)):
+ if (l4_proto == IPPROTO_SCTP)
+ m->ol_flags |= PKT_TX_SCTP_CKSUM;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
+ switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+ case VIRTIO_NET_HDR_GSO_TCPV4:
+ case VIRTIO_NET_HDR_GSO_TCPV6:
+ tcp_hdr = l4_hdr;
+ m->ol_flags |= PKT_TX_TCP_SEG;
+ m->tso_segsz = hdr->gso_size;
+ m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
+ break;
+ case VIRTIO_NET_HDR_GSO_UDP:
+ m->ol_flags |= PKT_TX_UDP_S
[dpdk-dev] [PATCH v2] net/virtio: fix unexpected event after reconnect
Event notification should be disabled after virtqueue initialization and
enabled by calling rte_eth_dev_rx_intr_enable later. When virtio user
device reconnecting to vhost, virtqueue_disable_intr should be called to
disable event notification.
Fixes: 6ebbf4109f35 ("net/virtio-user: fix packed ring server mode")
Cc: sta...@dpdk.org
Signed-off-by: Marvin Liu
---
v2: commit log update and cc stable
diff --git a/drivers/net/virtio/virtqueue.c b/drivers/net/virtio/virtqueue.c
index 0b4e3bf3e..02c8b9fc5 100644
--- a/drivers/net/virtio/virtqueue.c
+++ b/drivers/net/virtio/virtqueue.c
@@ -174,6 +174,7 @@ virtqueue_rxvq_reset_packed(struct virtqueue *vq)
vring_desc_init_packed(vq, size);
+ virtqueue_disable_intr(vq);
return 0;
}
@@ -210,5 +211,6 @@ virtqueue_txvq_reset_packed(struct virtqueue *vq)
vring_desc_init_packed(vq, size);
+ virtqueue_disable_intr(vq);
return 0;
}
--
2.17.1
[dpdk-dev] [PATCH] net/virtio: disable event suppression when reconnect
Event suppression should be disabled after virtqueue initialization. It can be enabled by calling rte_eth_dev_rx_intr_enable later. Signed-off-by: Marvin Liu diff --git a/drivers/net/virtio/virtqueue.c b/drivers/net/virtio/virtqueue.c index 408bba236a..2702e120ee 100644 --- a/drivers/net/virtio/virtqueue.c +++ b/drivers/net/virtio/virtqueue.c @@ -175,6 +175,7 @@ virtqueue_rxvq_reset_packed(struct virtqueue *vq) vring_desc_init_packed(vq, size); + virtqueue_disable_intr(vq); return 0; } @@ -211,5 +212,6 @@ virtqueue_txvq_reset_packed(struct virtqueue *vq) vring_desc_init_packed(vq, size); + virtqueue_disable_intr(vq); return 0; } -- 2.17.1
[dpdk-dev] [PATCH] net/virtio: remove duplicated runtime check
Runtime checks for vectorized datapath are organized into device
configuration function. Remove duplicated check in device init
function.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 49ccef12c7..312871cb48 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1965,10 +1965,8 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
PMD_DRV_LOG(INFO,
"building environment do not support packed
ring vectorized");
#else
- if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) {
- hw->use_vec_rx = 1;
- hw->use_vec_tx = 1;
- }
+ hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
#endif
}
}
--
2.17.1
[dpdk-dev] [PATCH v2] net/virtio: fix crash when device reconnecting
When doing virtio device initialization, virtqueues will be reset in
server mode if ring type is packed. It will cause issue because queues
have been freed in the beginning of device initialization.
Fix this issue by checking whether device has been initialized before
reset. If device hasn't been initialized, there's no need to reset
queues.
Fixes: 6ebbf4109f35 ("net/virtio-user: fix packed ring server mode")
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_user_ethdev.c
b/drivers/net/virtio/virtio_user_ethdev.c
index c54698ad1..55bd81f0b 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -89,7 +89,8 @@ virtio_user_server_reconnect(struct virtio_user_dev *dev)
dev->features &= dev->device_features;
/* For packed ring, resetting queues is required in reconnection. */
- if (vtpci_packed_queue(hw)) {
+ if (vtpci_packed_queue(hw) &&
+ (vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_DRIVER_OK)) {
PMD_INIT_LOG(NOTICE, "Packets on the fly will be dropped"
" when packed ring reconnecting.");
virtio_user_reset_queues_packed(eth_dev);
--
2.17.1
[dpdk-dev] [PATCH] net/virtio: fix 32-bit build with clang 6.0.0
Clang 6.0.0 will undefine function _mm512_maskz_set1_epi64 on i686
target. Fix it by replacing the function with _mm512_set4_epi64 when
doing 32-bit build.
Warning messasge during build:
../drivers/net/virtio/virtio_rxtx_packed_avx.c:385:19: warning:
implicit declaration of function '_mm512_maskz_set1_epi64' is invalid
in C99 [-Wimplicit-function-declaration]
Fixes: 8d35836e4a7c ("net/virtio: add vectorized packed ring Rx path")
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index d130d68bf..6a8214725 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -365,7 +365,12 @@ virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
return -1;
/* only care avail/used bits */
+#if defined(RTE_ARCH_I686)
+ __m512i v_mask = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
+ PACKED_FLAGS_MASK, 0x0);
+#else
__m512i v_mask = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
+#endif
desc_addr = &vq->vq_packed.ring.desc[id];
__m512i v_desc = _mm512_loadu_si512(desc_addr);
@@ -373,7 +378,12 @@ virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
__m512i v_used_flag = _mm512_setzero_si512();
if (vq->vq_packed.used_wrap_counter)
+#if defined(RTE_ARCH_I686)
+ v_used_flag = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
+ PACKED_FLAGS_MASK, 0x0);
+#else
v_used_flag = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
+#endif
/* Check all descs are used */
desc_stats = _mm512_cmpneq_epu64_mask(v_flag, v_used_flag);
--
2.17.1
[dpdk-dev] [PATCH v12 6/9] net/virtio: add vectorized packed ring Rx path
Optimize packed ring Rx path with SIMD instructions. Solution of
optimization is pretty like vhost, is that split path into batch and
single functions. Batch function is further optimized by AVX512
instructions.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index c9edb84ee..102b1deab 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -36,6 +36,41 @@ else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM)
$(CONFIG_RTE_ARCH_ARM64)),)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c
endif
+ifneq ($(FORCE_DISABLE_AVX512), y)
+ CC_AVX512_SUPPORT=\
+ $(shell $(CC) -march=native -dM -E - &1 | \
+ sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+ grep -q AVX512 && echo 1)
+endif
+
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_packed_avx.c
+
+ifeq ($(RTE_TOOLCHAIN), gcc)
+ifeq ($(shell test $(GCC_VERSION) -ge 83 && echo 1), 1)
+CFLAGS += -DVIRTIO_GCC_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), clang)
+ifeq ($(shell test $(CLANG_MAJOR_VERSION)$(CLANG_MINOR_VERSION) -ge 37 && echo
1), 1)
+CFLAGS += -DVIRTIO_CLANG_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), icc)
+ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 16 && echo 1), 1)
+CFLAGS += -DVIRTIO_ICC_UNROLL_PRAGMA
+endif
+endif
+
+CFLAGS_virtio_rxtx_packed_avx.o += -mavx512f -mavx512bw -mavx512vl
+ifeq ($(shell test $(GCC_VERSION) -ge 100 && echo 1), 1)
+CFLAGS_virtio_rxtx_packed_avx.o += -Wno-zero-length-bounds
+endif
+endif
+
ifeq ($(CONFIG_RTE_VIRTIO_USER),y)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_user.c
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel.c
diff --git a/drivers/net/virtio/meson.build b/drivers/net/virtio/meson.build
index 15150eea1..8e68c3039 100644
--- a/drivers/net/virtio/meson.build
+++ b/drivers/net/virtio/meson.build
@@ -9,6 +9,20 @@ sources += files('virtio_ethdev.c',
deps += ['kvargs', 'bus_pci']
if arch_subdir == 'x86'
+ if '-mno-avx512f' not in machine_args
+ if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+ cflags += ['-mavx512f', '-mavx512bw', '-mavx512vl']
+ cflags += ['-DCC_AVX512_SUPPORT']
+ if (toolchain == 'gcc' and
cc.version().version_compare('>=8.3.0'))
+ cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
+ elif (toolchain == 'clang' and
cc.version().version_compare('>=3.7.0'))
+ cflags += '-DVHOST_CLANG_UNROLL_PRAGMA'
+ elif (toolchain == 'icc' and
cc.version().version_compare('>=16.0.0'))
+ cflags += '-DVHOST_ICC_UNROLL_PRAGMA'
+ endif
+ sources += files('virtio_rxtx_packed_avx.c')
+ endif
+ endif
sources += files('virtio_rxtx_simple_sse.c')
elif arch_subdir == 'ppc'
sources += files('virtio_rxtx_simple_altivec.c')
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index febaf17a8..5c112cac7 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -105,6 +105,9 @@ uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct
rte_mbuf **tx_pkts,
uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index a549991aa..c50980c82 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2030,3 +2030,13 @@ virtio_xmit_pkts_inorder(void *tx_queue,
return nb_tx;
}
+
+#ifndef CC_AVX512_SUPPORT
+uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+ struct rte_mbuf **rx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
+#endif /* ifndef CC_AVX512_SUPPORT */
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
new file mode 100644
index 0..88831a786
--- /dev/null
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -0,0 +1,374 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 20
[dpdk-dev] [PATCH v12 9/9] doc: add packed vectorized path
Document packed virtqueue vectorized path selection logic in virtio net PMD. Signed-off-by: Marvin Liu Reviewed-by: Maxime Coquelin diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index fdd0790e0..226f4308d 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -482,6 +482,13 @@ according to below configuration: both negotiated, this path will be selected. #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. +#. Packed virtqueue vectorized Rx path: If building and running environment support + AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && + TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. +#. Packed virtqueue vectorized Tx path: If building and running environment support + AVX512 && in-order feature is negotiated && vectorized option enabled, + this path will be selected. Rx/Tx callbacks of each Virtio path ~~~ @@ -504,6 +511,8 @@ are shown in below table: Packed virtqueue non-meregable path virtio_recv_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order mergeable path virtio_recv_mergeable_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order non-mergeable path virtio_recv_pkts_packed virtio_xmit_pkts_packed + Packed virtqueue vectorized Rx path virtio_recv_pkts_packed_vec virtio_xmit_pkts_packed + Packed virtqueue vectorized Tx path virtio_recv_pkts_packed virtio_xmit_pkts_packed_vec = Virtio paths Support Status from Release to Release @@ -521,20 +530,22 @@ All virtio paths support status are shown in below table: .. table:: Virtio Paths and Releases - = = = - Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 - = = = - Split virtqueue mergeable path Y Y Y - Split virtqueue non-mergeable path Y Y Y - Split virtqueue vectorized Rx path Y Y Y - Split virtqueue simple Tx path Y N N - Split virtqueue in-order mergeable path Y Y - Split virtqueue in-order non-mergeable path Y Y - Packed virtqueue mergeable path Y - Packed virtqueue non-mergeable path Y - Packed virtqueue in-order mergeable path Y - Packed virtqueue in-order non-mergeable path Y - = = = + = = = === + Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 20.05 ~ + = = = === + Split virtqueue mergeable path Y Y Y Y + Split virtqueue non-mergeable path Y Y Y Y + Split virtqueue vectorized Rx path Y Y Y Y + Split virtqueue simple Tx path Y N N N + Split virtqueue in-order mergeable path Y Y Y + Split virtqueue in-order non-mergeable path Y Y Y + Packed virtqueue mergeable path Y Y + Packed virtqueue non-mergeable path Y Y + Packed virtqueue in-order mergeable path Y Y + Packed virtqueue in-order non-mergeable path Y Y + Packed virtqueue vectorized Rx path Y + Packed virtqueue vectorized Tx path Y + = = = === QEMU Support Status ~~~ -- 2.17.1
[dpdk-dev] [PATCH v12 5/9] net/virtio: reuse packed ring functions
Move offload, xmit cleanup and packed xmit enqueue function to header
file. These functions will be reused by packed ring vectorized path.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 84f4cf946..a549991aa 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -89,23 +89,6 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
dp->next = VQ_RING_DESC_CHAIN_END;
}
-static void
-vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
-{
- struct vq_desc_extra *dxp;
-
- dxp = &vq->vq_descx[id];
- vq->vq_free_cnt += dxp->ndescs;
-
- if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
- vq->vq_desc_head_idx = id;
- else
- vq->vq_descx[vq->vq_desc_tail_idx].next = id;
-
- vq->vq_desc_tail_idx = id;
- dxp->next = VQ_RING_DESC_CHAIN_END;
-}
-
void
virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
{
@@ -264,130 +247,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
return i;
}
-#ifndef DEFAULT_TX_FREE_THRESH
-#define DEFAULT_TX_FREE_THRESH 32
-#endif
-
-static void
-virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id, curr_id, free_cnt = 0;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- do {
- curr_id = used_idx;
- dxp = &vq->vq_descx[used_idx];
- used_idx += dxp->ndescs;
- free_cnt += dxp->ndescs;
- num -= dxp->ndescs;
- if (used_idx >= size) {
- used_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- } while (curr_id != id);
- }
- vq->vq_used_cons_idx = used_idx;
- vq->vq_free_cnt += free_cnt;
-}
-
-static void
-virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num-- && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- dxp = &vq->vq_descx[id];
- vq->vq_used_cons_idx += dxp->ndescs;
- if (vq->vq_used_cons_idx >= size) {
- vq->vq_used_cons_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- vq_ring_free_id_packed(vq, id);
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- used_idx = vq->vq_used_cons_idx;
- }
-}
-
-/* Cleanup from completed transmits. */
-static inline void
-virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
-{
- if (in_order)
- virtio_xmit_cleanup_inorder_packed(vq, num);
- else
- virtio_xmit_cleanup_normal_packed(vq, num);
-}
-
-static void
-virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
-{
- uint16_t i, used_idx, desc_idx;
- for (i = 0; i < num; i++) {
- struct vring_used_elem *uep;
- struct vq_desc_extra *dxp;
-
- used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries -
1));
- uep = &vq->vq_split.ring.used->ring[used_idx];
-
- desc_idx = (uint16_t) uep->id;
- dxp = &vq->vq_descx[desc_idx];
- vq->vq_used_cons_idx++;
- vq_ring_free_chain(vq, desc_idx);
-
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- }
-}
-
-/* Cleanup from completed inorder transmits. */
-static __rte_always_inline void
-virtio_xmit_cleanup_
[dpdk-dev] [PATCH v12 7/9] net/virtio: add vectorized packed ring Tx path
Optimize packed ring Tx path like Rx path. Split Tx path into batch and
single Tx functions. Batch function is further optimized by AVX512
instructions.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index 5c112cac7..b7d52d497 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -108,6 +108,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index c50980c82..050541a10 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2039,4 +2039,12 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
{
return 0;
}
+
+uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+ struct rte_mbuf **tx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
#endif /* ifndef CC_AVX512_SUPPORT */
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 88831a786..d130d68bf 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -23,6 +23,24 @@
#define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
FLAGS_BITS_OFFSET)
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+/* id bits offset in packed ring desc higher 64bits */
+#define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+/* net hdr short size mask */
+#define NET_HDR_MASK 0x3F
+
#define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
sizeof(struct vring_packed_desc))
#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
@@ -60,6 +78,221 @@ virtio_update_batch_stats(struct virtnet_stats *stats,
stats->bytes += pkt_len4;
}
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+ struct rte_mbuf **tx_pkts)
+{
+ struct virtqueue *vq = txvq->vq;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
+ uint16_t idx = vq->vq_avail_idx;
+ struct virtio_net_hdr *hdr;
+ struct vq_desc_extra *dxp;
+ uint16_t i, cmp;
+
+ if (vq->vq_avail_idx & PACKED_BATCH_MASK)
+ return -1;
+
+ if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+ return -1;
+
+ /* Load four mbufs rearm data */
+ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+ RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
+ __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
+ *tx_pkts[2]->rearm_data,
+ *tx_pkts[1]->rearm_data,
+ *tx_pkts[0]->rearm_data);
+
+ /* refcnt=1 and nb_segs=1 */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+ __m256i head_rooms = _mm256_set1_epi16(head_size);
+
+ /* Check refcnt and nb_segs */
+ const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
+ cmp = _mm256_mask_cmpneq_epu16_mask(mask, mbufs, mbuf_ref);
+ if (unlikely(cmp))
+ return -1;
+
+ /* Check headroom is enough */
+ const __mmask16 data_mask = 0x1 | 0x1 << 4 | 0x1 << 8 | 0x1 << 12;
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
+ offsetof(struct rte_mbuf, rearm_data));
+ cmp = _mm256_mask_cmplt_epu16_mask(data_mask, mbufs, head_rooms);
+ if (unlikely(cmp))
+ return -1;
+
+ virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ dxp = &vq->vq_descx[idx + i];
+ dxp->ndescs = 1;
+ dxp->cookie = tx_pkts[i];
+ }
+
+ virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ tx_pkts[i]->data_off -= head_size;
+ tx_pkts[i]->data_len += head_size;
+ }
+
+
[dpdk-dev] [PATCH v12 8/9] net/virtio: add election for vectorized path
Rewrite vectorized path selection logic. Default setting comes from
vectorized devarg, then checks each criteria.
Packed ring vectorized path need:
AVX512F and required extensions are supported by compiler and host
VERSION_1 and IN_ORDER features are negotiated
mergeable feature is not negotiated
LRO offloading is disabled
Split ring vectorized rx path need:
mergeable and IN_ORDER features are not negotiated
LRO, chksum and vlan strip offloadings are disabled
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 0a69a4db1..e86d4e08f 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1523,9 +1523,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
if (vtpci_packed_queue(hw)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring %s Tx path on port %u",
- hw->use_inorder_tx ? "inorder" : "standard",
+ hw->use_vec_tx ? "vectorized" : "standard",
eth_dev->data->port_id);
- eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
+ if (hw->use_vec_tx)
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
+ else
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
} else {
if (hw->use_inorder_tx) {
PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on
port %u",
@@ -1539,7 +1542,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
}
if (vtpci_packed_queue(hw)) {
- if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO,
+ "virtio: using packed ring vectorized Rx path
on port %u",
+ eth_dev->data->port_id);
+ eth_dev->rx_pkt_burst =
+ &virtio_recv_pkts_packed_vec;
+ } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring mergeable buffer Rx
path on port %u",
eth_dev->data->port_id);
@@ -1952,8 +1961,17 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
goto err_virtio_init;
if (vectorized) {
- if (!vtpci_packed_queue(hw))
+ if (!vtpci_packed_queue(hw)) {
+ hw->use_vec_rx = 1;
+ } else {
+#if !defined(CC_AVX512_SUPPORT)
+ PMD_DRV_LOG(INFO,
+ "building environment do not support packed
ring vectorized");
+#else
hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
+#endif
+ }
}
hw->opened = true;
@@ -2288,31 +2306,66 @@ virtio_dev_configure(struct rte_eth_dev *dev)
return -EBUSY;
}
- if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
- hw->use_inorder_tx = 1;
- hw->use_inorder_rx = 1;
- hw->use_vec_rx = 0;
- }
-
if (vtpci_packed_queue(hw)) {
+#if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT)
+ if ((hw->use_vec_rx || hw->use_vec_tx) &&
+ (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
+!vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+!vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized path for
requirements not met");
+ hw->use_vec_rx = 0;
+ hw->use_vec_tx = 0;
+ }
+#else
hw->use_vec_rx = 0;
- hw->use_inorder_rx = 0;
- }
+ hw->use_vec_tx = 0;
+#endif
+
+ if (hw->use_vec_rx) {
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
mrg_rxbuf enabled");
+ hw->use_vec_rx = 0;
+ }
+ if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
TCP_LRO enabled");
+ hw->use_vec_rx = 0;
+ }
+ }
+ } else {
+
[dpdk-dev] [PATCH v12 4/9] net/virtio-user: add vectorized devarg
Add new devarg for virtio user device vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index a67774e91..fdd0790e0 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -424,6 +424,12 @@ Below devargs are supported by the virtio-user vdev:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfers to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
Virtio paths Selection and Usage
diff --git a/drivers/net/virtio/virtio_user_ethdev.c
b/drivers/net/virtio/virtio_user_ethdev.c
index 150a8d987..40ad786cc 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -452,6 +452,8 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_PACKED_VQ,
#define VIRTIO_USER_ARG_SPEED "speed"
VIRTIO_USER_ARG_SPEED,
+#define VIRTIO_USER_ARG_VECTORIZED "vectorized"
+ VIRTIO_USER_ARG_VECTORIZED,
NULL
};
@@ -559,6 +561,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t mrg_rxbuf = 1;
uint64_t in_order = 1;
uint64_t packed_vq = 0;
+ uint64_t vectorized = 0;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
@@ -675,6 +678,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
+ &get_integer_arg, &vectorized) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+VIRTIO_USER_ARG_VECTORIZED);
+ goto end;
+ }
+ }
+
if (queues > 1 && cq == 0) {
PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
goto end;
@@ -727,6 +739,9 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vectorized)
+ hw->use_vec_rx = 1;
+
rte_eth_dev_probing_finish(eth_dev);
ret = 0;
@@ -785,4 +800,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
"mrg_rxbuf=<0|1> "
"in_order=<0|1> "
"packed_vq=<0|1> "
- "speed=");
+ "speed= "
+ "vectorized=<0|1>");
--
2.17.1
[dpdk-dev] [PATCH v12 2/9] net/virtio: inorder should depend on feature bit
Ring initialization is different when inorder feature negotiated. This
action should dependent on negotiated feature bits.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 94ba7a3ec..e450477e8 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -989,6 +989,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
struct rte_mbuf *m;
uint16_t desc_idx;
int error, nbufs, i;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
PMD_INIT_FUNC_TRACE();
@@ -1018,7 +1019,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
virtio_rxq_rearm_vec(rxvq);
nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
}
- } else if (hw->use_inorder_rx) {
+ } else if (!vtpci_packed_queue(vq->hw) && in_order) {
if ((!virtqueue_full(vq))) {
uint16_t free_cnt = vq->vq_free_cnt;
struct rte_mbuf *pkts[free_cnt];
@@ -1133,7 +1134,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
PMD_INIT_FUNC_TRACE();
if (!vtpci_packed_queue(hw)) {
- if (hw->use_inorder_tx)
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER))
vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
}
@@ -2046,7 +2047,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_tx = 0;
- bool in_order = hw->use_inorder_tx;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
return nb_tx;
--
2.17.1
[dpdk-dev] [PATCH v12 3/9] net/virtio: add vectorized devarg
Previously, virtio split ring vectorized path was enabled by default.
This is not suitable for everyone because that path dose not follow
virtio spec. Add new devarg for virtio vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 6286286db..a67774e91 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -363,6 +363,13 @@ Below devargs are supported by the PCI virtio driver:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfers to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
+
Below devargs are supported by the virtio-user vdev:
#. ``path``:
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 37766cbb6..0a69a4db1 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -48,7 +48,8 @@ static int virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev);
static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
- uint32_t *speed);
+ uint32_t *speed,
+ int *vectorized);
static int virtio_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info);
static int virtio_dev_link_update(struct rte_eth_dev *dev,
@@ -1551,8 +1552,8 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
}
} else {
- if (hw->use_simple_rx) {
- PMD_INIT_LOG(INFO, "virtio: using simple Rx path on
port %u",
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on
port %u",
eth_dev->data->port_id);
eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
} else if (hw->use_inorder_rx) {
@@ -1886,6 +1887,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
{
struct virtio_hw *hw = eth_dev->data->dev_private;
uint32_t speed = SPEED_UNKNOWN;
+ int vectorized = 0;
int ret;
if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
@@ -1912,7 +1914,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
ret = virtio_dev_devargs_parse(eth_dev->device->devargs,
-NULL, &speed);
+NULL, &speed, &vectorized);
if (ret < 0)
return ret;
hw->speed = speed;
@@ -1949,6 +1951,11 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
if (ret < 0)
goto err_virtio_init;
+ if (vectorized) {
+ if (!vtpci_packed_queue(hw))
+ hw->use_vec_rx = 1;
+ }
+
hw->opened = true;
return 0;
@@ -2021,9 +2028,20 @@ virtio_dev_speed_capa_get(uint32_t speed)
}
}
+static int vectorized_check_handler(__rte_unused const char *key,
+ const char *value, void *ret_val)
+{
+ if (strcmp(value, "1") == 0)
+ *(int *)ret_val = 1;
+ else
+ *(int *)ret_val = 0;
+
+ return 0;
+}
#define VIRTIO_ARG_SPEED "speed"
#define VIRTIO_ARG_VDPA "vdpa"
+#define VIRTIO_ARG_VECTORIZED "vectorized"
static int
@@ -2045,7 +2063,7 @@ link_speed_handler(const char *key __rte_unused,
static int
virtio_dev_devargs_parse(struct rte_devargs *devargs, int *vdpa,
- uint32_t *speed)
+ uint32_t *speed, int *vectorized)
{
struct rte_kvargs *kvlist;
int ret = 0;
@@ -2081,6 +2099,18 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
}
}
+ if (vectorized &&
+ rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
+ if (ret < 0) {
+ PMD_INIT_LOG(ERR, "Failed to parse %s",
+ VIRTIO_ARG_VECTORIZED);
+ goto exit;
+ }
+ }
+
exit:
rte_kvargs_free(kvlist);
return ret;
@@ -2092,7 +2122,8 @@ static int eth_virtio_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused,
int vdpa = 0;
int ret = 0;
- ret = virtio_dev_devargs_parse(pci_dev->device.devargs, &vdpa, NULL);
+ ret = virtio_dev_dev
[dpdk-dev] [PATCH v12 0/9] add packed ring vectorized path
This patch set introduced vectorized path for packed ring. The size of packed ring descriptor is 16Bytes. Four batched descriptors are just placed into one cacheline. AVX512 instructions can well handle this kind of data. Packed ring TX path can fully transformed into vectorized path. Packed ring Rx path can be vectorized when requirements met(LRO and mergeable disabled). New device parameter "vectorized" will be introduced in this patch set. This parameter will be workable for both virtio device and virtio user vdev. It will also unify split and packed ring vectorized path default setting. Path election logic will check dependencies of vectorized path. Packed ring vectorized path is dependent on building/running environment and features like IN_ORDER and VERSION_1 enabled, MRG and LRO disabled. If vectorized path is not supported, will fallback to normal path. v12: * eliminate weak symbols in data path * remove desc extra padding which can impact normal path * fix enqueue address invalid v11: * fix i686 build warnings * fix typo in doc v10: * reuse packed ring xmit cleanup v9: * replace RTE_LIBRTE_VIRTIO_INC_VECTOR with vectorized devarg * reorder patch sequence v8: * fix meson build error on ubuntu16.04 and suse15 v7: * default vectorization is disabled * compilation time check dependency on rte_mbuf structure * offsets are calcuated when compiling * remove useless barrier as descs are batched store&load * vindex of scatter is directly set * some comments updates * enable vectorized path in meson build v6: * fix issue when size not power of 2 v5: * remove cpuflags definition as required extensions always come with AVX512F on x86_64 * inorder actions should depend on feature bit * check ring type in rx queue setup * rewrite some commit logs * fix some checkpatch warnings v4: * rename 'packed_vec' to 'vectorized', also used in split ring * add RTE_LIBRTE_VIRTIO_INC_VECTOR config for virtio ethdev * check required AVX512 extensions cpuflags * combine split and packed ring datapath selection logic * remove limitation that size must power of two * clear 12Bytes virtio_net_hdr v3: * remove virtio_net_hdr array for better performance * disable 'packed_vec' by default v2: * more function blocks replaced by vector instructions * clean virtio_net_hdr by vector instruction * allow header room size change * add 'packed_vec' option in virtio_user vdev * fix build not check whether AVX512 enabled * doc update Tested-by: Wang, Yinan Marvin Liu (9): net/virtio: add Rx free threshold setting net/virtio: inorder should depend on feature bit net/virtio: add vectorized devarg net/virtio-user: add vectorized devarg net/virtio: reuse packed ring functions net/virtio: add vectorized packed ring Rx path net/virtio: add vectorized packed ring Tx path net/virtio: add election for vectorized path doc: add packed vectorized path doc/guides/nics/virtio.rst | 52 +- drivers/net/virtio/Makefile | 35 ++ drivers/net/virtio/meson.build | 14 + drivers/net/virtio/virtio_ethdev.c | 142 - drivers/net/virtio/virtio_ethdev.h | 6 + drivers/net/virtio/virtio_pci.h | 3 +- drivers/net/virtio/virtio_rxtx.c| 351 ++- drivers/net/virtio/virtio_rxtx_packed_avx.c | 607 drivers/net/virtio/virtio_user_ethdev.c | 32 +- drivers/net/virtio/virtqueue.c | 7 +- drivers/net/virtio/virtqueue.h | 304 ++ 11 files changed, 1199 insertions(+), 354 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c -- 2.17.1
[dpdk-dev] [PATCH v12 1/9] net/virtio: add Rx free threshold setting
Introduce free threshold setting in Rx queue, its default value is 32.
Limit the threshold size to multiple of four as only vectorized packed
Rx function will utilize it. Virtio driver will rearm Rx queue when
more than rx_free_thresh descs were dequeued.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 060410577..94ba7a3ec 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -936,6 +936,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
struct virtio_hw *hw = dev->data->dev_private;
struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
struct virtnet_rx *rxvq;
+ uint16_t rx_free_thresh;
PMD_INIT_FUNC_TRACE();
@@ -944,6 +945,28 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
return -EINVAL;
}
+ rx_free_thresh = rx_conf->rx_free_thresh;
+ if (rx_free_thresh == 0)
+ rx_free_thresh =
+ RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
+
+ if (rx_free_thresh & 0x3) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+
+ if (rx_free_thresh >= vq->vq_nentries) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the "
+ "number of RX entries (%u)."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ vq->vq_nentries,
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+ vq->vq_free_thresh = rx_free_thresh;
+
if (nb_desc == 0 || nb_desc > vq->vq_nentries)
nb_desc = vq->vq_nentries;
vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 58ad7309a..6301c56b2 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -18,6 +18,8 @@
struct rte_mbuf;
+#define DEFAULT_RX_FREE_THRESH 32
+
/*
* Per virtio_ring.h in Linux.
* For virtio_pci on SMP, we don't need to order with respect to MMIO
--
2.17.1
[dpdk-dev] [PATCH v4 1/2] vhost: utilize dpdk dynamic memory allocator
Replace dynamic memory allocator with dpdk memory allocator.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index bd1be0104..79fcb9d19 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -191,7 +191,7 @@ vhost_backend_cleanup(struct virtio_net *dev)
dev->mem = NULL;
}
- free(dev->guest_pages);
+ rte_free(dev->guest_pages);
dev->guest_pages = NULL;
if (dev->log_addr) {
@@ -903,11 +903,12 @@ add_one_guest_page(struct virtio_net *dev, uint64_t
guest_phys_addr,
if (dev->nr_guest_pages == dev->max_guest_pages) {
dev->max_guest_pages *= 2;
old_pages = dev->guest_pages;
- dev->guest_pages = realloc(dev->guest_pages,
- dev->max_guest_pages * sizeof(*page));
- if (!dev->guest_pages) {
+ dev->guest_pages = rte_realloc(dev->guest_pages,
+ dev->max_guest_pages * sizeof(*page),
+ RTE_CACHE_LINE_SIZE);
+ if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
- free(old_pages);
+ rte_free(old_pages);
return -1;
}
}
@@ -1062,10 +1063,12 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
vhost_user_iotlb_flush_all(dev->virtqueue[i]);
dev->nr_guest_pages = 0;
- if (!dev->guest_pages) {
+ if (dev->guest_pages == NULL) {
dev->max_guest_pages = 8;
- dev->guest_pages = malloc(dev->max_guest_pages *
- sizeof(struct guest_page));
+ dev->guest_pages = rte_zmalloc(NULL,
+ dev->max_guest_pages *
+ sizeof(struct guest_page),
+ RTE_CACHE_LINE_SIZE);
if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR,
"(%d) failed to allocate memory "
--
2.17.1
[dpdk-dev] [PATCH v4 2/2] vhost: binary search address mapping table
If Tx zero copy enabled, gpa to hpa mapping table is updated one by
one. This will harm performance when guest memory backend using 2M
hugepages. Now utilize binary search to find the entry in mapping
table, meanwhile set the threshold to 256 entries for linear search.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 507dbf214..998e133ad 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -546,20 +546,48 @@ extern int vhost_data_log_level;
#define MAX_VHOST_DEVICE 1024
extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
+#define VHOST_BINARY_SEARCH_THRESH 256
+
+static __rte_always_inline int guest_page_addrcmp(const void *p1,
+ const void *p2)
+{
+ const struct guest_page *page1 = (const struct guest_page *)p1;
+ const struct guest_page *page2 = (const struct guest_page *)p2;
+
+ if (page1->guest_phys_addr > page2->guest_phys_addr)
+ return 1;
+ if (page1->guest_phys_addr < page2->guest_phys_addr)
+ return -1;
+
+ return 0;
+}
+
/* Convert guest physical address to host physical address */
static __rte_always_inline rte_iova_t
gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
{
uint32_t i;
struct guest_page *page;
-
- for (i = 0; i < dev->nr_guest_pages; i++) {
- page = &dev->guest_pages[i];
-
- if (gpa >= page->guest_phys_addr &&
- gpa + size < page->guest_phys_addr + page->size) {
- return gpa - page->guest_phys_addr +
- page->host_phys_addr;
+ struct guest_page key;
+
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ key.guest_phys_addr = gpa;
+ page = bsearch(&key, dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ if (page) {
+ if (gpa + size < page->guest_phys_addr + page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
+ }
+ } else {
+ for (i = 0; i < dev->nr_guest_pages; i++) {
+ page = &dev->guest_pages[i];
+
+ if (gpa >= page->guest_phys_addr &&
+ gpa + size < page->guest_phys_addr +
+ page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
}
}
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 79fcb9d19..15e50d27d 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -965,6 +965,12 @@ add_guest_pages(struct virtio_net *dev, struct
rte_vhost_mem_region *reg,
reg_size -= size;
}
+ /* sort guest page array if over binary search threshold */
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ qsort((void *)dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ }
+
return 0;
}
--
2.17.1
[dpdk-dev] [PATCH v4 1/2] vhost: utilize dpdk dynamic memory allocator
Replace dynamic memory allocator with dpdk memory allocator.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index bd1be0104..79fcb9d19 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -191,7 +191,7 @@ vhost_backend_cleanup(struct virtio_net *dev)
dev->mem = NULL;
}
- free(dev->guest_pages);
+ rte_free(dev->guest_pages);
dev->guest_pages = NULL;
if (dev->log_addr) {
@@ -903,11 +903,12 @@ add_one_guest_page(struct virtio_net *dev, uint64_t
guest_phys_addr,
if (dev->nr_guest_pages == dev->max_guest_pages) {
dev->max_guest_pages *= 2;
old_pages = dev->guest_pages;
- dev->guest_pages = realloc(dev->guest_pages,
- dev->max_guest_pages * sizeof(*page));
- if (!dev->guest_pages) {
+ dev->guest_pages = rte_realloc(dev->guest_pages,
+ dev->max_guest_pages * sizeof(*page),
+ RTE_CACHE_LINE_SIZE);
+ if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
- free(old_pages);
+ rte_free(old_pages);
return -1;
}
}
@@ -1062,10 +1063,12 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
vhost_user_iotlb_flush_all(dev->virtqueue[i]);
dev->nr_guest_pages = 0;
- if (!dev->guest_pages) {
+ if (dev->guest_pages == NULL) {
dev->max_guest_pages = 8;
- dev->guest_pages = malloc(dev->max_guest_pages *
- sizeof(struct guest_page));
+ dev->guest_pages = rte_zmalloc(NULL,
+ dev->max_guest_pages *
+ sizeof(struct guest_page),
+ RTE_CACHE_LINE_SIZE);
if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR,
"(%d) failed to allocate memory "
--
2.17.1
[dpdk-dev] [PATCH v4 2/2] vhost: utilize dpdk dynamic memory allocator
Replace dynamic memory allocator with dpdk memory allocator.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index bd1be0104..79fcb9d19 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -191,7 +191,7 @@ vhost_backend_cleanup(struct virtio_net *dev)
dev->mem = NULL;
}
- free(dev->guest_pages);
+ rte_free(dev->guest_pages);
dev->guest_pages = NULL;
if (dev->log_addr) {
@@ -903,11 +903,12 @@ add_one_guest_page(struct virtio_net *dev, uint64_t
guest_phys_addr,
if (dev->nr_guest_pages == dev->max_guest_pages) {
dev->max_guest_pages *= 2;
old_pages = dev->guest_pages;
- dev->guest_pages = realloc(dev->guest_pages,
- dev->max_guest_pages * sizeof(*page));
- if (!dev->guest_pages) {
+ dev->guest_pages = rte_realloc(dev->guest_pages,
+ dev->max_guest_pages * sizeof(*page),
+ RTE_CACHE_LINE_SIZE);
+ if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
- free(old_pages);
+ rte_free(old_pages);
return -1;
}
}
@@ -1062,10 +1063,12 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
vhost_user_iotlb_flush_all(dev->virtqueue[i]);
dev->nr_guest_pages = 0;
- if (!dev->guest_pages) {
+ if (dev->guest_pages == NULL) {
dev->max_guest_pages = 8;
- dev->guest_pages = malloc(dev->max_guest_pages *
- sizeof(struct guest_page));
+ dev->guest_pages = rte_zmalloc(NULL,
+ dev->max_guest_pages *
+ sizeof(struct guest_page),
+ RTE_CACHE_LINE_SIZE);
if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR,
"(%d) failed to allocate memory "
--
2.17.1
[dpdk-dev] [PATCH v4 2/2] vhost: binary search address mapping table
If Tx zero copy enabled, gpa to hpa mapping table is updated one by
one. This will harm performance when guest memory backend using 2M
hugepages. Now utilize binary search to find the entry in mapping
table, meanwhile set the threshold to 256 entries for linear search.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 507dbf214..998e133ad 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -546,20 +546,48 @@ extern int vhost_data_log_level;
#define MAX_VHOST_DEVICE 1024
extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
+#define VHOST_BINARY_SEARCH_THRESH 256
+
+static __rte_always_inline int guest_page_addrcmp(const void *p1,
+ const void *p2)
+{
+ const struct guest_page *page1 = (const struct guest_page *)p1;
+ const struct guest_page *page2 = (const struct guest_page *)p2;
+
+ if (page1->guest_phys_addr > page2->guest_phys_addr)
+ return 1;
+ if (page1->guest_phys_addr < page2->guest_phys_addr)
+ return -1;
+
+ return 0;
+}
+
/* Convert guest physical address to host physical address */
static __rte_always_inline rte_iova_t
gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
{
uint32_t i;
struct guest_page *page;
-
- for (i = 0; i < dev->nr_guest_pages; i++) {
- page = &dev->guest_pages[i];
-
- if (gpa >= page->guest_phys_addr &&
- gpa + size < page->guest_phys_addr + page->size) {
- return gpa - page->guest_phys_addr +
- page->host_phys_addr;
+ struct guest_page key;
+
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ key.guest_phys_addr = gpa;
+ page = bsearch(&key, dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ if (page) {
+ if (gpa + size < page->guest_phys_addr + page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
+ }
+ } else {
+ for (i = 0; i < dev->nr_guest_pages; i++) {
+ page = &dev->guest_pages[i];
+
+ if (gpa >= page->guest_phys_addr &&
+ gpa + size < page->guest_phys_addr +
+ page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
}
}
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 79fcb9d19..15e50d27d 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -965,6 +965,12 @@ add_guest_pages(struct virtio_net *dev, struct
rte_vhost_mem_region *reg,
reg_size -= size;
}
+ /* sort guest page array if over binary search threshold */
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ qsort((void *)dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ }
+
return 0;
}
--
2.17.1
[dpdk-dev] [PATCH v4 1/2] net/virtio: add support Virtio link speed feature
From: Ivan Dyukov
This patch adds a support of VIRTIO_NET_F_SPEED_DUPLEX feature
for virtio driver.
There are two ways to specify speed of the link:
'speed' devarg
negotiate speed from qemu via VIRTIO_NET_F_SPEED_DUPLEX
The highest priority is devarg. If devarg is not specified,
drivers tries to negotiate it from qemu.
Signed-off-by: Ivan Dyukov
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 5058990d1..37766cbb6 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1662,7 +1662,8 @@ virtio_configure_intr(struct rte_eth_dev *dev)
return 0;
}
-
+#define SPEED_UNKNOWN0x
+#define DUPLEX_UNKNOWN 0xff
/* reset device and renegotiate features if needed */
static int
virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
@@ -1718,6 +1719,25 @@ virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t
req_features)
hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
+ if (hw->speed == SPEED_UNKNOWN) {
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) {
+ config = &local_config;
+ vtpci_read_dev_config(hw,
+ offsetof(struct virtio_net_config, speed),
+ &config->speed, sizeof(config->speed));
+ vtpci_read_dev_config(hw,
+ offsetof(struct virtio_net_config, duplex),
+ &config->duplex, sizeof(config->duplex));
+ hw->speed = config->speed;
+ hw->duplex = config->duplex;
+ }
+ }
+ if (hw->speed == SPEED_UNKNOWN)
+ hw->speed = ETH_SPEED_NUM_10G;
+ if (hw->duplex == DUPLEX_UNKNOWN)
+ hw->duplex = ETH_LINK_FULL_DUPLEX;
+ PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d",
+ hw->speed, hw->duplex);
if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
config = &local_config;
@@ -1865,7 +1885,7 @@ int
eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
{
struct virtio_hw *hw = eth_dev->data->dev_private;
- uint32_t speed = ETH_SPEED_NUM_10G;
+ uint32_t speed = SPEED_UNKNOWN;
int ret;
if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
@@ -2450,7 +2470,7 @@ virtio_dev_link_update(struct rte_eth_dev *dev,
__rte_unused int wait_to_complet
struct virtio_hw *hw = dev->data->dev_private;
memset(&link, 0, sizeof(link));
- link.link_duplex = ETH_LINK_FULL_DUPLEX;
+ link.link_duplex = hw->duplex;
link.link_speed = hw->speed;
link.link_autoneg = ETH_LINK_AUTONEG;
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index cd8947656..febaf17a8 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -37,7 +37,8 @@
1ULL << VIRTIO_F_RING_PACKED | \
1ULL << VIRTIO_F_IOMMU_PLATFORM | \
1ULL << VIRTIO_F_ORDER_PLATFORM | \
-1ULL << VIRTIO_F_NOTIFICATION_DATA)
+1ULL << VIRTIO_F_NOTIFICATION_DATA | \
+1ULL << VIRTIO_NET_F_SPEED_DUPLEX)
#define VIRTIO_PMD_SUPPORTED_GUEST_FEATURES\
(VIRTIO_PMD_DEFAULT_GUEST_FEATURES |\
diff --git a/drivers/net/virtio/virtio_pci.h b/drivers/net/virtio/virtio_pci.h
index ed98e11c3..bd89357e4 100644
--- a/drivers/net/virtio/virtio_pci.h
+++ b/drivers/net/virtio/virtio_pci.h
@@ -141,6 +141,9 @@ struct virtnet_ctl;
*/
#define VIRTIO_F_NOTIFICATION_DATA 38
+/* Device set linkspeed and duplex */
+#define VIRTIO_NET_F_SPEED_DUPLEX 63
+
/* The Guest publishes the used index for which it expects an interrupt
* at the end of the avail ring. Host should ignore the avail->flags field. */
/* The Host publishes the avail index for which it expects a kick
@@ -260,6 +263,7 @@ struct virtio_hw {
uint8_t mac_addr[RTE_ETHER_ADDR_LEN];
uint32_tnotify_off_multiplier;
uint32_tspeed; /* link speed in MB */
+ uint8_t duplex;
uint8_t *isr;
uint16_t*notify_base;
struct virtio_pci_common_cfg *common_cfg;
@@ -306,6 +310,18 @@ struct virtio_net_config {
uint16_t status;
uint16_t max_virtqueue_pairs;
uint16_t mtu;
+ /*
+* speed, in units of 1Mb. All values 0 to INT_MAX are legal.
+* Any other value stands for unknown.
+*/
+ uint32_t speed;
+ /*
+* 0x00 - half duplex
+* 0x01 - full duplex
+* Any other value stands for unknown.
+*/
+ uint8_t duplex;
+
} __attribute__((packed));
/*
--
2.17.1
[dpdk-dev] [PATCH v3 2/2] vhost: binary search address mapping table
If Tx zero copy enabled, gpa to hpa mapping table is updated one by
one. This will harm performance when guest memory backend using 2M
hugepages. Now utilize binary search to find the entry in mapping
table, meanwhile set threshold to 256 entries for linear search.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/Makefile b/lib/librte_vhost/Makefile
index e592795f2..8769afaad 100644
--- a/lib/librte_vhost/Makefile
+++ b/lib/librte_vhost/Makefile
@@ -10,7 +10,7 @@ EXPORT_MAP := rte_vhost_version.map
CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3
CFLAGS += -I vhost_user
-CFLAGS += -fno-strict-aliasing
+CFLAGS += -fno-strict-aliasing -Wno-maybe-uninitialized
LDLIBS += -lpthread
ifeq ($(RTE_TOOLCHAIN), gcc)
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 507dbf214..a0fee39d5 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -546,20 +546,46 @@ extern int vhost_data_log_level;
#define MAX_VHOST_DEVICE 1024
extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
+#define VHOST_BINARY_SEARCH_THRESH 256
+static int guest_page_addrcmp(const void *p1, const void *p2)
+{
+ const struct guest_page *page1 = (const struct guest_page *)p1;
+ const struct guest_page *page2 = (const struct guest_page *)p2;
+
+ if (page1->guest_phys_addr > page2->guest_phys_addr)
+ return 1;
+ if (page1->guest_phys_addr < page2->guest_phys_addr)
+ return -1;
+
+ return 0;
+}
+
/* Convert guest physical address to host physical address */
static __rte_always_inline rte_iova_t
gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size)
{
uint32_t i;
struct guest_page *page;
-
- for (i = 0; i < dev->nr_guest_pages; i++) {
- page = &dev->guest_pages[i];
-
- if (gpa >= page->guest_phys_addr &&
- gpa + size < page->guest_phys_addr + page->size) {
- return gpa - page->guest_phys_addr +
- page->host_phys_addr;
+ struct guest_page key;
+
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ key.guest_phys_addr = gpa;
+ page = bsearch(&key, dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ if (page) {
+ if (gpa + size < page->guest_phys_addr + page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
+ }
+ } else {
+ for (i = 0; i < dev->nr_guest_pages; i++) {
+ page = &dev->guest_pages[i];
+
+ if (gpa >= page->guest_phys_addr &&
+ gpa + size < page->guest_phys_addr +
+ page->size)
+ return gpa - page->guest_phys_addr +
+ page->host_phys_addr;
}
}
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index 79fcb9d19..15e50d27d 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -965,6 +965,12 @@ add_guest_pages(struct virtio_net *dev, struct
rte_vhost_mem_region *reg,
reg_size -= size;
}
+ /* sort guest page array if over binary search threshold */
+ if (dev->nr_guest_pages >= VHOST_BINARY_SEARCH_THRESH) {
+ qsort((void *)dev->guest_pages, dev->nr_guest_pages,
+ sizeof(struct guest_page), guest_page_addrcmp);
+ }
+
return 0;
}
--
2.17.1
[dpdk-dev] [PATCH v3 1/2] vhost: utilize dpdk dynamic memory allocator
Replace dynamic memory allocator with dpdk memory allocator.
Signed-off-by: Marvin Liu
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index bd1be0104..79fcb9d19 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -191,7 +191,7 @@ vhost_backend_cleanup(struct virtio_net *dev)
dev->mem = NULL;
}
- free(dev->guest_pages);
+ rte_free(dev->guest_pages);
dev->guest_pages = NULL;
if (dev->log_addr) {
@@ -903,11 +903,12 @@ add_one_guest_page(struct virtio_net *dev, uint64_t
guest_phys_addr,
if (dev->nr_guest_pages == dev->max_guest_pages) {
dev->max_guest_pages *= 2;
old_pages = dev->guest_pages;
- dev->guest_pages = realloc(dev->guest_pages,
- dev->max_guest_pages * sizeof(*page));
- if (!dev->guest_pages) {
+ dev->guest_pages = rte_realloc(dev->guest_pages,
+ dev->max_guest_pages * sizeof(*page),
+ RTE_CACHE_LINE_SIZE);
+ if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR, "cannot realloc guest_pages\n");
- free(old_pages);
+ rte_free(old_pages);
return -1;
}
}
@@ -1062,10 +1063,12 @@ vhost_user_set_mem_table(struct virtio_net **pdev,
struct VhostUserMsg *msg,
vhost_user_iotlb_flush_all(dev->virtqueue[i]);
dev->nr_guest_pages = 0;
- if (!dev->guest_pages) {
+ if (dev->guest_pages == NULL) {
dev->max_guest_pages = 8;
- dev->guest_pages = malloc(dev->max_guest_pages *
- sizeof(struct guest_page));
+ dev->guest_pages = rte_zmalloc(NULL,
+ dev->max_guest_pages *
+ sizeof(struct guest_page),
+ RTE_CACHE_LINE_SIZE);
if (dev->guest_pages == NULL) {
VHOST_LOG_CONFIG(ERR,
"(%d) failed to allocate memory "
--
2.17.1
[dpdk-dev] [PATCH v11 9/9] doc: add packed vectorized path
Document packed virtqueue vectorized path selection logic in virtio net PMD. Signed-off-by: Marvin Liu Reviewed-by: Maxime Coquelin diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index fdd0790e0..226f4308d 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -482,6 +482,13 @@ according to below configuration: both negotiated, this path will be selected. #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. +#. Packed virtqueue vectorized Rx path: If building and running environment support + AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && + TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. +#. Packed virtqueue vectorized Tx path: If building and running environment support + AVX512 && in-order feature is negotiated && vectorized option enabled, + this path will be selected. Rx/Tx callbacks of each Virtio path ~~~ @@ -504,6 +511,8 @@ are shown in below table: Packed virtqueue non-meregable path virtio_recv_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order mergeable path virtio_recv_mergeable_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order non-mergeable path virtio_recv_pkts_packed virtio_xmit_pkts_packed + Packed virtqueue vectorized Rx path virtio_recv_pkts_packed_vec virtio_xmit_pkts_packed + Packed virtqueue vectorized Tx path virtio_recv_pkts_packed virtio_xmit_pkts_packed_vec = Virtio paths Support Status from Release to Release @@ -521,20 +530,22 @@ All virtio paths support status are shown in below table: .. table:: Virtio Paths and Releases - = = = - Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 - = = = - Split virtqueue mergeable path Y Y Y - Split virtqueue non-mergeable path Y Y Y - Split virtqueue vectorized Rx path Y Y Y - Split virtqueue simple Tx path Y N N - Split virtqueue in-order mergeable path Y Y - Split virtqueue in-order non-mergeable path Y Y - Packed virtqueue mergeable path Y - Packed virtqueue non-mergeable path Y - Packed virtqueue in-order mergeable path Y - Packed virtqueue in-order non-mergeable path Y - = = = + = = = === + Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 20.05 ~ + = = = === + Split virtqueue mergeable path Y Y Y Y + Split virtqueue non-mergeable path Y Y Y Y + Split virtqueue vectorized Rx path Y Y Y Y + Split virtqueue simple Tx path Y N N N + Split virtqueue in-order mergeable path Y Y Y + Split virtqueue in-order non-mergeable path Y Y Y + Packed virtqueue mergeable path Y Y + Packed virtqueue non-mergeable path Y Y + Packed virtqueue in-order mergeable path Y Y + Packed virtqueue in-order non-mergeable path Y Y + Packed virtqueue vectorized Rx path Y + Packed virtqueue vectorized Tx path Y + = = = === QEMU Support Status ~~~ -- 2.17.1
[dpdk-dev] [PATCH v11 5/9] net/virtio: reuse packed ring functions
Move offload, xmit cleanup and packed xmit enqueue function to header
file. These functions will be reused by packed ring vectorized path.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 84f4cf946..a549991aa 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -89,23 +89,6 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
dp->next = VQ_RING_DESC_CHAIN_END;
}
-static void
-vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
-{
- struct vq_desc_extra *dxp;
-
- dxp = &vq->vq_descx[id];
- vq->vq_free_cnt += dxp->ndescs;
-
- if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
- vq->vq_desc_head_idx = id;
- else
- vq->vq_descx[vq->vq_desc_tail_idx].next = id;
-
- vq->vq_desc_tail_idx = id;
- dxp->next = VQ_RING_DESC_CHAIN_END;
-}
-
void
virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
{
@@ -264,130 +247,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
return i;
}
-#ifndef DEFAULT_TX_FREE_THRESH
-#define DEFAULT_TX_FREE_THRESH 32
-#endif
-
-static void
-virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id, curr_id, free_cnt = 0;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- do {
- curr_id = used_idx;
- dxp = &vq->vq_descx[used_idx];
- used_idx += dxp->ndescs;
- free_cnt += dxp->ndescs;
- num -= dxp->ndescs;
- if (used_idx >= size) {
- used_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- } while (curr_id != id);
- }
- vq->vq_used_cons_idx = used_idx;
- vq->vq_free_cnt += free_cnt;
-}
-
-static void
-virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num-- && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- dxp = &vq->vq_descx[id];
- vq->vq_used_cons_idx += dxp->ndescs;
- if (vq->vq_used_cons_idx >= size) {
- vq->vq_used_cons_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- vq_ring_free_id_packed(vq, id);
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- used_idx = vq->vq_used_cons_idx;
- }
-}
-
-/* Cleanup from completed transmits. */
-static inline void
-virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
-{
- if (in_order)
- virtio_xmit_cleanup_inorder_packed(vq, num);
- else
- virtio_xmit_cleanup_normal_packed(vq, num);
-}
-
-static void
-virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
-{
- uint16_t i, used_idx, desc_idx;
- for (i = 0; i < num; i++) {
- struct vring_used_elem *uep;
- struct vq_desc_extra *dxp;
-
- used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries -
1));
- uep = &vq->vq_split.ring.used->ring[used_idx];
-
- desc_idx = (uint16_t) uep->id;
- dxp = &vq->vq_descx[desc_idx];
- vq->vq_used_cons_idx++;
- vq_ring_free_chain(vq, desc_idx);
-
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- }
-}
-
-/* Cleanup from completed inorder transmits. */
-static __rte_always_inline void
-virtio_xmit_cleanup_
[dpdk-dev] [PATCH v11 7/9] net/virtio: add vectorized packed ring Tx path
Optimize packed ring Tx path like Rx path. Split Tx path into batch and
single Tx functions. Batch function is further optimized by AVX512
instructions.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index 5c112cac7..b7d52d497 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -108,6 +108,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 534562cca..460e9d4a2 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2038,3 +2038,11 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
{
return 0;
}
+
+__rte_weak uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+ struct rte_mbuf **tx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 88831a786..a7358c768 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -23,6 +23,24 @@
#define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
FLAGS_BITS_OFFSET)
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+/* id bits offset in packed ring desc higher 64bits */
+#define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+/* net hdr short size mask */
+#define NET_HDR_MASK 0x3F
+
#define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
sizeof(struct vring_packed_desc))
#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
@@ -60,6 +78,237 @@ virtio_update_batch_stats(struct virtnet_stats *stats,
stats->bytes += pkt_len4;
}
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+ struct rte_mbuf **tx_pkts)
+{
+ struct virtqueue *vq = txvq->vq;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
+ uint16_t idx = vq->vq_avail_idx;
+ struct virtio_net_hdr *hdr;
+ uint16_t i, cmp;
+
+ if (vq->vq_avail_idx & PACKED_BATCH_MASK)
+ return -1;
+
+ if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+ return -1;
+
+ /* Load four mbufs rearm data */
+ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+ RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
+ __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
+ *tx_pkts[2]->rearm_data,
+ *tx_pkts[1]->rearm_data,
+ *tx_pkts[0]->rearm_data);
+
+ /* refcnt=1 and nb_segs=1 */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+ __m256i head_rooms = _mm256_set1_epi16(head_size);
+
+ /* Check refcnt and nb_segs */
+ const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
+ cmp = _mm256_mask_cmpneq_epu16_mask(mask, mbufs, mbuf_ref);
+ if (unlikely(cmp))
+ return -1;
+
+ /* Check headroom is enough */
+ const __mmask16 data_mask = 0x1 | 0x1 << 4 | 0x1 << 8 | 0x1 << 12;
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
+ offsetof(struct rte_mbuf, rearm_data));
+ cmp = _mm256_mask_cmplt_epu16_mask(data_mask, mbufs, head_rooms);
+ if (unlikely(cmp))
+ return -1;
+
+ __m512i v_descx = _mm512_set_epi64(0x1, (uintptr_t)tx_pkts[3],
+ 0x1, (uintptr_t)tx_pkts[2],
+ 0x1, (uintptr_t)tx_pkts[1],
+ 0x1, (uintptr_t)tx_pkts[0]);
+
+ _mm512_storeu_si512((void *)&vq->vq_descx[idx], v_descx);
+
+ virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ tx_pkts[i]->data_off -= hea
[dpdk-dev] [PATCH v11 6/9] net/virtio: add vectorized packed ring Rx path
Optimize packed ring Rx path with SIMD instructions. Solution of
optimization is pretty like vhost, is that split path into batch and
single functions. Batch function is further optimized by AVX512
instructions. Also pad desc extra structure to 16 bytes aligned, thus
four elements will be saved in one batch.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index c9edb84ee..102b1deab 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -36,6 +36,41 @@ else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM)
$(CONFIG_RTE_ARCH_ARM64)),)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c
endif
+ifneq ($(FORCE_DISABLE_AVX512), y)
+ CC_AVX512_SUPPORT=\
+ $(shell $(CC) -march=native -dM -E - &1 | \
+ sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+ grep -q AVX512 && echo 1)
+endif
+
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_packed_avx.c
+
+ifeq ($(RTE_TOOLCHAIN), gcc)
+ifeq ($(shell test $(GCC_VERSION) -ge 83 && echo 1), 1)
+CFLAGS += -DVIRTIO_GCC_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), clang)
+ifeq ($(shell test $(CLANG_MAJOR_VERSION)$(CLANG_MINOR_VERSION) -ge 37 && echo
1), 1)
+CFLAGS += -DVIRTIO_CLANG_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), icc)
+ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 16 && echo 1), 1)
+CFLAGS += -DVIRTIO_ICC_UNROLL_PRAGMA
+endif
+endif
+
+CFLAGS_virtio_rxtx_packed_avx.o += -mavx512f -mavx512bw -mavx512vl
+ifeq ($(shell test $(GCC_VERSION) -ge 100 && echo 1), 1)
+CFLAGS_virtio_rxtx_packed_avx.o += -Wno-zero-length-bounds
+endif
+endif
+
ifeq ($(CONFIG_RTE_VIRTIO_USER),y)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_user.c
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel.c
diff --git a/drivers/net/virtio/meson.build b/drivers/net/virtio/meson.build
index 15150eea1..8e68c3039 100644
--- a/drivers/net/virtio/meson.build
+++ b/drivers/net/virtio/meson.build
@@ -9,6 +9,20 @@ sources += files('virtio_ethdev.c',
deps += ['kvargs', 'bus_pci']
if arch_subdir == 'x86'
+ if '-mno-avx512f' not in machine_args
+ if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+ cflags += ['-mavx512f', '-mavx512bw', '-mavx512vl']
+ cflags += ['-DCC_AVX512_SUPPORT']
+ if (toolchain == 'gcc' and
cc.version().version_compare('>=8.3.0'))
+ cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
+ elif (toolchain == 'clang' and
cc.version().version_compare('>=3.7.0'))
+ cflags += '-DVHOST_CLANG_UNROLL_PRAGMA'
+ elif (toolchain == 'icc' and
cc.version().version_compare('>=16.0.0'))
+ cflags += '-DVHOST_ICC_UNROLL_PRAGMA'
+ endif
+ sources += files('virtio_rxtx_packed_avx.c')
+ endif
+ endif
sources += files('virtio_rxtx_simple_sse.c')
elif arch_subdir == 'ppc'
sources += files('virtio_rxtx_simple_altivec.c')
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index febaf17a8..5c112cac7 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -105,6 +105,9 @@ uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct
rte_mbuf **tx_pkts,
uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index a549991aa..534562cca 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2030,3 +2030,11 @@ virtio_xmit_pkts_inorder(void *tx_queue,
return nb_tx;
}
+
+__rte_weak uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+ struct rte_mbuf **rx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
new file mode 100644
index 0..88831a786
--- /dev/null
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -0,0 +1,374 @@
+/* SPDX-License-Identif
[dpdk-dev] [PATCH v11 8/9] net/virtio: add election for vectorized path
Rewrite vectorized path selection logic. Default setting comes from
vectorized devarg, then checks each criteria.
Packed ring vectorized path need:
AVX512F and required extensions are supported by compiler and host
VERSION_1 and IN_ORDER features are negotiated
mergeable feature is not negotiated
LRO offloading is disabled
Split ring vectorized rx path need:
mergeable and IN_ORDER features are not negotiated
LRO, chksum and vlan strip offloadings are disabled
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 0a69a4db1..088d0e45e 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1523,9 +1523,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
if (vtpci_packed_queue(hw)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring %s Tx path on port %u",
- hw->use_inorder_tx ? "inorder" : "standard",
+ hw->use_vec_tx ? "vectorized" : "standard",
eth_dev->data->port_id);
- eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
+ if (hw->use_vec_tx)
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
+ else
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
} else {
if (hw->use_inorder_tx) {
PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on
port %u",
@@ -1539,7 +1542,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
}
if (vtpci_packed_queue(hw)) {
- if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO,
+ "virtio: using packed ring vectorized Rx path
on port %u",
+ eth_dev->data->port_id);
+ eth_dev->rx_pkt_burst =
+ &virtio_recv_pkts_packed_vec;
+ } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring mergeable buffer Rx
path on port %u",
eth_dev->data->port_id);
@@ -1952,8 +1961,17 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
goto err_virtio_init;
if (vectorized) {
- if (!vtpci_packed_queue(hw))
+ if (!vtpci_packed_queue(hw)) {
+ hw->use_vec_rx = 1;
+ } else {
+#if !defined(CC_AVX512_SUPPORT)
+ PMD_DRV_LOG(INFO,
+ "building environment do not support packed
ring vectorized");
+#else
hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
+#endif
+ }
}
hw->opened = true;
@@ -2288,31 +2306,61 @@ virtio_dev_configure(struct rte_eth_dev *dev)
return -EBUSY;
}
- if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
- hw->use_inorder_tx = 1;
- hw->use_inorder_rx = 1;
- hw->use_vec_rx = 0;
- }
-
if (vtpci_packed_queue(hw)) {
- hw->use_vec_rx = 0;
- hw->use_inorder_rx = 0;
- }
+ if ((hw->use_vec_rx || hw->use_vec_tx) &&
+ (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
+!vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+!vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized path for
requirements not met");
+ hw->use_vec_rx = 0;
+ hw->use_vec_tx = 0;
+ }
+ if (hw->use_vec_rx) {
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
mrg_rxbuf enabled");
+ hw->use_vec_rx = 0;
+ }
+
+ if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
TCP_LRO enabled");
+ hw->use_vec_rx = 0;
+ }
+ }
+ } else {
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
+ hw->use_inorder_tx = 1;
+
[dpdk-dev] [PATCH v11 3/9] net/virtio: add vectorized devarg
Previously, virtio split ring vectorized path was enabled by default.
This is not suitable for everyone because that path dose not follow
virtio spec. Add new devarg for virtio vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 6286286db..a67774e91 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -363,6 +363,13 @@ Below devargs are supported by the PCI virtio driver:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfers to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
+
Below devargs are supported by the virtio-user vdev:
#. ``path``:
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 37766cbb6..0a69a4db1 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -48,7 +48,8 @@ static int virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev);
static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
- uint32_t *speed);
+ uint32_t *speed,
+ int *vectorized);
static int virtio_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info);
static int virtio_dev_link_update(struct rte_eth_dev *dev,
@@ -1551,8 +1552,8 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
}
} else {
- if (hw->use_simple_rx) {
- PMD_INIT_LOG(INFO, "virtio: using simple Rx path on
port %u",
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on
port %u",
eth_dev->data->port_id);
eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
} else if (hw->use_inorder_rx) {
@@ -1886,6 +1887,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
{
struct virtio_hw *hw = eth_dev->data->dev_private;
uint32_t speed = SPEED_UNKNOWN;
+ int vectorized = 0;
int ret;
if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
@@ -1912,7 +1914,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
ret = virtio_dev_devargs_parse(eth_dev->device->devargs,
-NULL, &speed);
+NULL, &speed, &vectorized);
if (ret < 0)
return ret;
hw->speed = speed;
@@ -1949,6 +1951,11 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
if (ret < 0)
goto err_virtio_init;
+ if (vectorized) {
+ if (!vtpci_packed_queue(hw))
+ hw->use_vec_rx = 1;
+ }
+
hw->opened = true;
return 0;
@@ -2021,9 +2028,20 @@ virtio_dev_speed_capa_get(uint32_t speed)
}
}
+static int vectorized_check_handler(__rte_unused const char *key,
+ const char *value, void *ret_val)
+{
+ if (strcmp(value, "1") == 0)
+ *(int *)ret_val = 1;
+ else
+ *(int *)ret_val = 0;
+
+ return 0;
+}
#define VIRTIO_ARG_SPEED "speed"
#define VIRTIO_ARG_VDPA "vdpa"
+#define VIRTIO_ARG_VECTORIZED "vectorized"
static int
@@ -2045,7 +2063,7 @@ link_speed_handler(const char *key __rte_unused,
static int
virtio_dev_devargs_parse(struct rte_devargs *devargs, int *vdpa,
- uint32_t *speed)
+ uint32_t *speed, int *vectorized)
{
struct rte_kvargs *kvlist;
int ret = 0;
@@ -2081,6 +2099,18 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
}
}
+ if (vectorized &&
+ rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
+ if (ret < 0) {
+ PMD_INIT_LOG(ERR, "Failed to parse %s",
+ VIRTIO_ARG_VECTORIZED);
+ goto exit;
+ }
+ }
+
exit:
rte_kvargs_free(kvlist);
return ret;
@@ -2092,7 +2122,8 @@ static int eth_virtio_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused,
int vdpa = 0;
int ret = 0;
- ret = virtio_dev_devargs_parse(pci_dev->device.devargs, &vdpa, NULL);
+ ret = virtio_dev_dev
[dpdk-dev] [PATCH v11 4/9] net/virtio-user: add vectorized devarg
Add new devarg for virtio user device vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index a67774e91..fdd0790e0 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -424,6 +424,12 @@ Below devargs are supported by the virtio-user vdev:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfers to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
Virtio paths Selection and Usage
diff --git a/drivers/net/virtio/virtio_user_ethdev.c
b/drivers/net/virtio/virtio_user_ethdev.c
index 150a8d987..40ad786cc 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -452,6 +452,8 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_PACKED_VQ,
#define VIRTIO_USER_ARG_SPEED "speed"
VIRTIO_USER_ARG_SPEED,
+#define VIRTIO_USER_ARG_VECTORIZED "vectorized"
+ VIRTIO_USER_ARG_VECTORIZED,
NULL
};
@@ -559,6 +561,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t mrg_rxbuf = 1;
uint64_t in_order = 1;
uint64_t packed_vq = 0;
+ uint64_t vectorized = 0;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
@@ -675,6 +678,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
+ &get_integer_arg, &vectorized) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+VIRTIO_USER_ARG_VECTORIZED);
+ goto end;
+ }
+ }
+
if (queues > 1 && cq == 0) {
PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
goto end;
@@ -727,6 +739,9 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vectorized)
+ hw->use_vec_rx = 1;
+
rte_eth_dev_probing_finish(eth_dev);
ret = 0;
@@ -785,4 +800,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
"mrg_rxbuf=<0|1> "
"in_order=<0|1> "
"packed_vq=<0|1> "
- "speed=");
+ "speed= "
+ "vectorized=<0|1>");
--
2.17.1
[dpdk-dev] [PATCH v11 1/9] net/virtio: add Rx free threshold setting
Introduce free threshold setting in Rx queue, its default value is 32.
Limit the threshold size to multiple of four as only vectorized packed
Rx function will utilize it. Virtio driver will rearm Rx queue when
more than rx_free_thresh descs were dequeued.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 060410577..94ba7a3ec 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -936,6 +936,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
struct virtio_hw *hw = dev->data->dev_private;
struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
struct virtnet_rx *rxvq;
+ uint16_t rx_free_thresh;
PMD_INIT_FUNC_TRACE();
@@ -944,6 +945,28 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
return -EINVAL;
}
+ rx_free_thresh = rx_conf->rx_free_thresh;
+ if (rx_free_thresh == 0)
+ rx_free_thresh =
+ RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
+
+ if (rx_free_thresh & 0x3) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+
+ if (rx_free_thresh >= vq->vq_nentries) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the "
+ "number of RX entries (%u)."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ vq->vq_nentries,
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+ vq->vq_free_thresh = rx_free_thresh;
+
if (nb_desc == 0 || nb_desc > vq->vq_nentries)
nb_desc = vq->vq_nentries;
vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 58ad7309a..6301c56b2 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -18,6 +18,8 @@
struct rte_mbuf;
+#define DEFAULT_RX_FREE_THRESH 32
+
/*
* Per virtio_ring.h in Linux.
* For virtio_pci on SMP, we don't need to order with respect to MMIO
--
2.17.1
[dpdk-dev] [PATCH v11 2/9] net/virtio: inorder should depend on feature bit
Ring initialization is different when inorder feature negotiated. This
action should dependent on negotiated feature bits.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 94ba7a3ec..e450477e8 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -989,6 +989,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
struct rte_mbuf *m;
uint16_t desc_idx;
int error, nbufs, i;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
PMD_INIT_FUNC_TRACE();
@@ -1018,7 +1019,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
virtio_rxq_rearm_vec(rxvq);
nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
}
- } else if (hw->use_inorder_rx) {
+ } else if (!vtpci_packed_queue(vq->hw) && in_order) {
if ((!virtqueue_full(vq))) {
uint16_t free_cnt = vq->vq_free_cnt;
struct rte_mbuf *pkts[free_cnt];
@@ -1133,7 +1134,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
PMD_INIT_FUNC_TRACE();
if (!vtpci_packed_queue(hw)) {
- if (hw->use_inorder_tx)
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER))
vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
}
@@ -2046,7 +2047,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_tx = 0;
- bool in_order = hw->use_inorder_tx;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
return nb_tx;
--
2.17.1
[dpdk-dev] [PATCH v11 0/9] add packed ring vectorized path
This patch set introduced vectorized path for packed ring. The size of packed ring descriptor is 16Bytes. Four batched descriptors are just placed into one cacheline. AVX512 instructions can well handle this kind of data. Packed ring TX path can fully transformed into vectorized path. Packed ring Rx path can be vectorized when requirements met(LRO and mergeable disabled). New option RTE_LIBRTE_VIRTIO_INC_VECTOR will be introduced in this patch set. This option will unify split and packed ring vectorized path default setting. Meanwhile user can specify whether enable vectorized path at runtime by 'vectorized' parameter of virtio user vdev. v11: * fix i686 build warnings * fix typo in doc v10: * reuse packed ring xmit cleanup v9: * replace RTE_LIBRTE_VIRTIO_INC_VECTOR with vectorized devarg * reorder patch sequence v8: * fix meson build error on ubuntu16.04 and suse15 v7: * default vectorization is disabled * compilation time check dependency on rte_mbuf structure * offsets are calcuated when compiling * remove useless barrier as descs are batched store&load * vindex of scatter is directly set * some comments updates * enable vectorized path in meson build v6: * fix issue when size not power of 2 v5: * remove cpuflags definition as required extensions always come with AVX512F on x86_64 * inorder actions should depend on feature bit * check ring type in rx queue setup * rewrite some commit logs * fix some checkpatch warnings v4: * rename 'packed_vec' to 'vectorized', also used in split ring * add RTE_LIBRTE_VIRTIO_INC_VECTOR config for virtio ethdev * check required AVX512 extensions cpuflags * combine split and packed ring datapath selection logic * remove limitation that size must power of two * clear 12Bytes virtio_net_hdr v3: * remove virtio_net_hdr array for better performance * disable 'packed_vec' by default v2: * more function blocks replaced by vector instructions * clean virtio_net_hdr by vector instruction * allow header room size change * add 'packed_vec' option in virtio_user vdev * fix build not check whether AVX512 enabled * doc update Tested-by: Wang, Yinan Marvin Liu (9): net/virtio: add Rx free threshold setting net/virtio: inorder should depend on feature bit net/virtio: add vectorized devarg net/virtio-user: add vectorized devarg net/virtio: reuse packed ring functions net/virtio: add vectorized packed ring Rx path net/virtio: add vectorized packed ring Tx path net/virtio: add election for vectorized path doc: add packed vectorized path doc/guides/nics/virtio.rst | 52 +- drivers/net/virtio/Makefile | 35 ++ drivers/net/virtio/meson.build | 14 + drivers/net/virtio/virtio_ethdev.c | 137 - drivers/net/virtio/virtio_ethdev.h | 6 + drivers/net/virtio/virtio_pci.h | 3 +- drivers/net/virtio/virtio_rxtx.c| 349 ++- drivers/net/virtio/virtio_rxtx_packed_avx.c | 623 drivers/net/virtio/virtio_user_ethdev.c | 32 +- drivers/net/virtio/virtqueue.c | 7 +- drivers/net/virtio/virtqueue.h | 307 +- 11 files changed, 1210 insertions(+), 355 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c -- 2.17.1
[dpdk-dev] [PATCH v10 6/9] net/virtio: add vectorized packed ring Rx path
Optimize packed ring Rx path with SIMD instructions. Solution of
optimization is pretty like vhost, is that split path into batch and
single functions. Batch function is further optimized by AVX512
instructions. Also pad desc extra structure to 16 bytes aligned, thus
four elements will be saved in one batch.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index c9edb84ee..102b1deab 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -36,6 +36,41 @@ else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM)
$(CONFIG_RTE_ARCH_ARM64)),)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c
endif
+ifneq ($(FORCE_DISABLE_AVX512), y)
+ CC_AVX512_SUPPORT=\
+ $(shell $(CC) -march=native -dM -E - &1 | \
+ sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+ grep -q AVX512 && echo 1)
+endif
+
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_packed_avx.c
+
+ifeq ($(RTE_TOOLCHAIN), gcc)
+ifeq ($(shell test $(GCC_VERSION) -ge 83 && echo 1), 1)
+CFLAGS += -DVIRTIO_GCC_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), clang)
+ifeq ($(shell test $(CLANG_MAJOR_VERSION)$(CLANG_MINOR_VERSION) -ge 37 && echo
1), 1)
+CFLAGS += -DVIRTIO_CLANG_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), icc)
+ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 16 && echo 1), 1)
+CFLAGS += -DVIRTIO_ICC_UNROLL_PRAGMA
+endif
+endif
+
+CFLAGS_virtio_rxtx_packed_avx.o += -mavx512f -mavx512bw -mavx512vl
+ifeq ($(shell test $(GCC_VERSION) -ge 100 && echo 1), 1)
+CFLAGS_virtio_rxtx_packed_avx.o += -Wno-zero-length-bounds
+endif
+endif
+
ifeq ($(CONFIG_RTE_VIRTIO_USER),y)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_user.c
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel.c
diff --git a/drivers/net/virtio/meson.build b/drivers/net/virtio/meson.build
index 15150eea1..8e68c3039 100644
--- a/drivers/net/virtio/meson.build
+++ b/drivers/net/virtio/meson.build
@@ -9,6 +9,20 @@ sources += files('virtio_ethdev.c',
deps += ['kvargs', 'bus_pci']
if arch_subdir == 'x86'
+ if '-mno-avx512f' not in machine_args
+ if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+ cflags += ['-mavx512f', '-mavx512bw', '-mavx512vl']
+ cflags += ['-DCC_AVX512_SUPPORT']
+ if (toolchain == 'gcc' and
cc.version().version_compare('>=8.3.0'))
+ cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
+ elif (toolchain == 'clang' and
cc.version().version_compare('>=3.7.0'))
+ cflags += '-DVHOST_CLANG_UNROLL_PRAGMA'
+ elif (toolchain == 'icc' and
cc.version().version_compare('>=16.0.0'))
+ cflags += '-DVHOST_ICC_UNROLL_PRAGMA'
+ endif
+ sources += files('virtio_rxtx_packed_avx.c')
+ endif
+ endif
sources += files('virtio_rxtx_simple_sse.c')
elif arch_subdir == 'ppc'
sources += files('virtio_rxtx_simple_altivec.c')
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index febaf17a8..5c112cac7 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -105,6 +105,9 @@ uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct
rte_mbuf **tx_pkts,
uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index a549991aa..534562cca 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2030,3 +2030,11 @@ virtio_xmit_pkts_inorder(void *tx_queue,
return nb_tx;
}
+
+__rte_weak uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+ struct rte_mbuf **rx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
new file mode 100644
index 0..8a7b459eb
--- /dev/null
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -0,0 +1,374 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+
[dpdk-dev] [PATCH v10 9/9] doc: add packed vectorized path
Document packed virtqueue vectorized path selection logic in virtio net PMD. Signed-off-by: Marvin Liu Reviewed-by: Maxime Coquelin diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index d59add23e..dbcf49ae1 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -482,6 +482,13 @@ according to below configuration: both negotiated, this path will be selected. #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. +#. Packed virtqueue vectorized Rx path: If building and running environment support + AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && + TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. +#. Packed virtqueue vectorized Tx path: If building and running environment support + AVX512 && in-order feature is negotiated && vectorized option enabled, + this path will be selected. Rx/Tx callbacks of each Virtio path ~~~ @@ -504,6 +511,8 @@ are shown in below table: Packed virtqueue non-meregable path virtio_recv_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order mergeable path virtio_recv_mergeable_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order non-mergeable path virtio_recv_pkts_packed virtio_xmit_pkts_packed + Packed virtqueue vectorized Rx path virtio_recv_pkts_packed_vec virtio_xmit_pkts_packed + Packed virtqueue vectorized Tx path virtio_recv_pkts_packed virtio_xmit_pkts_packed_vec = Virtio paths Support Status from Release to Release @@ -521,20 +530,22 @@ All virtio paths support status are shown in below table: .. table:: Virtio Paths and Releases - = = = - Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 - = = = - Split virtqueue mergeable path Y Y Y - Split virtqueue non-mergeable path Y Y Y - Split virtqueue vectorized Rx path Y Y Y - Split virtqueue simple Tx path Y N N - Split virtqueue in-order mergeable path Y Y - Split virtqueue in-order non-mergeable path Y Y - Packed virtqueue mergeable path Y - Packed virtqueue non-mergeable path Y - Packed virtqueue in-order mergeable path Y - Packed virtqueue in-order non-mergeable path Y - = = = + = = = === + Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 20.05 ~ + = = = === + Split virtqueue mergeable path Y Y Y Y + Split virtqueue non-mergeable path Y Y Y Y + Split virtqueue vectorized Rx path Y Y Y Y + Split virtqueue simple Tx path Y N N N + Split virtqueue in-order mergeable path Y Y Y + Split virtqueue in-order non-mergeable path Y Y Y + Packed virtqueue mergeable path Y Y + Packed virtqueue non-mergeable path Y Y + Packed virtqueue in-order mergeable path Y Y + Packed virtqueue in-order non-mergeable path Y Y + Packed virtqueue vectorized Rx path Y + Packed virtqueue vectorized Tx path Y + = = = === QEMU Support Status ~~~ -- 2.17.1
[dpdk-dev] [PATCH v10 5/9] net/virtio: reuse packed ring functions
Move offload, xmit cleanup and packed xmit enqueue function to header
file. These functions will be reused by packed ring vectorized path.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 84f4cf946..a549991aa 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -89,23 +89,6 @@ vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
dp->next = VQ_RING_DESC_CHAIN_END;
}
-static void
-vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
-{
- struct vq_desc_extra *dxp;
-
- dxp = &vq->vq_descx[id];
- vq->vq_free_cnt += dxp->ndescs;
-
- if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
- vq->vq_desc_head_idx = id;
- else
- vq->vq_descx[vq->vq_desc_tail_idx].next = id;
-
- vq->vq_desc_tail_idx = id;
- dxp->next = VQ_RING_DESC_CHAIN_END;
-}
-
void
virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
{
@@ -264,130 +247,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
return i;
}
-#ifndef DEFAULT_TX_FREE_THRESH
-#define DEFAULT_TX_FREE_THRESH 32
-#endif
-
-static void
-virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id, curr_id, free_cnt = 0;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- do {
- curr_id = used_idx;
- dxp = &vq->vq_descx[used_idx];
- used_idx += dxp->ndescs;
- free_cnt += dxp->ndescs;
- num -= dxp->ndescs;
- if (used_idx >= size) {
- used_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- } while (curr_id != id);
- }
- vq->vq_used_cons_idx = used_idx;
- vq->vq_free_cnt += free_cnt;
-}
-
-static void
-virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
-{
- uint16_t used_idx, id;
- uint16_t size = vq->vq_nentries;
- struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
- struct vq_desc_extra *dxp;
-
- used_idx = vq->vq_used_cons_idx;
- /* desc_is_used has a load-acquire or rte_cio_rmb inside
-* and wait for used desc in virtqueue.
-*/
- while (num-- && desc_is_used(&desc[used_idx], vq)) {
- id = desc[used_idx].id;
- dxp = &vq->vq_descx[id];
- vq->vq_used_cons_idx += dxp->ndescs;
- if (vq->vq_used_cons_idx >= size) {
- vq->vq_used_cons_idx -= size;
- vq->vq_packed.used_wrap_counter ^= 1;
- }
- vq_ring_free_id_packed(vq, id);
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- used_idx = vq->vq_used_cons_idx;
- }
-}
-
-/* Cleanup from completed transmits. */
-static inline void
-virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
-{
- if (in_order)
- virtio_xmit_cleanup_inorder_packed(vq, num);
- else
- virtio_xmit_cleanup_normal_packed(vq, num);
-}
-
-static void
-virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
-{
- uint16_t i, used_idx, desc_idx;
- for (i = 0; i < num; i++) {
- struct vring_used_elem *uep;
- struct vq_desc_extra *dxp;
-
- used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries -
1));
- uep = &vq->vq_split.ring.used->ring[used_idx];
-
- desc_idx = (uint16_t) uep->id;
- dxp = &vq->vq_descx[desc_idx];
- vq->vq_used_cons_idx++;
- vq_ring_free_chain(vq, desc_idx);
-
- if (dxp->cookie != NULL) {
- rte_pktmbuf_free(dxp->cookie);
- dxp->cookie = NULL;
- }
- }
-}
-
-/* Cleanup from completed inorder transmits. */
-static __rte_always_inline void
-virtio_xmit_cleanup_inorder(struct virtqueue
[dpdk-dev] [PATCH v10 8/9] net/virtio: add election for vectorized path
Rewrite vectorized path selection logic. Default setting comes from
vectorized devarg, then checks each criteria.
Packed ring vectorized path need:
AVX512F and required extensions are supported by compiler and host
VERSION_1 and IN_ORDER features are negotiated
mergeable feature is not negotiated
LRO offloading is disabled
Split ring vectorized rx path need:
mergeable and IN_ORDER features are not negotiated
LRO, chksum and vlan strip offloadings are disabled
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 0a69a4db1..f8ff41d99 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1523,9 +1523,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
if (vtpci_packed_queue(hw)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring %s Tx path on port %u",
- hw->use_inorder_tx ? "inorder" : "standard",
+ hw->use_vec_tx ? "vectorized" : "standard",
eth_dev->data->port_id);
- eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
+ if (hw->use_vec_tx)
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
+ else
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
} else {
if (hw->use_inorder_tx) {
PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on
port %u",
@@ -1539,7 +1542,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
}
if (vtpci_packed_queue(hw)) {
- if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO,
+ "virtio: using packed ring vectorized Rx path
on port %u",
+ eth_dev->data->port_id);
+ eth_dev->rx_pkt_burst =
+ &virtio_recv_pkts_packed_vec;
+ } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring mergeable buffer Rx
path on port %u",
eth_dev->data->port_id);
@@ -1952,8 +1961,17 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
goto err_virtio_init;
if (vectorized) {
- if (!vtpci_packed_queue(hw))
+ if (!vtpci_packed_queue(hw)) {
+ hw->use_vec_rx = 1;
+ } else {
+#if !defined(CC_AVX512_SUPPORT)
+ PMD_DRV_LOG(INFO,
+ "building environment do not support packed
ring vectorized");
+#else
hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
+#endif
+ }
}
hw->opened = true;
@@ -2102,8 +2120,8 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs, int
*vdpa,
if (vectorized &&
rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
ret = rte_kvargs_process(kvlist,
- VIRTIO_ARG_VECTORIZED,
- vectorized_check_handler, vectorized);
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
if (ret < 0) {
PMD_INIT_LOG(ERR, "Failed to parse %s",
VIRTIO_ARG_VECTORIZED);
@@ -2288,31 +2306,61 @@ virtio_dev_configure(struct rte_eth_dev *dev)
return -EBUSY;
}
- if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
- hw->use_inorder_tx = 1;
- hw->use_inorder_rx = 1;
- hw->use_vec_rx = 0;
- }
-
if (vtpci_packed_queue(hw)) {
- hw->use_vec_rx = 0;
- hw->use_inorder_rx = 0;
- }
+ if ((hw->use_vec_rx || hw->use_vec_tx) &&
+ (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
+!vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+!vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized path for
requirements not met");
+ hw->use_vec_rx = 0;
+ hw->use_vec_tx = 0;
+ }
+ if (hw->use_vec_rx) {
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ PMD_DR
[dpdk-dev] [PATCH v10 7/9] net/virtio: add vectorized packed ring Tx path
Optimize packed ring Tx path like Rx path. Split Tx path into batch and
single Tx functions. Batch function is further optimized by AVX512
instructions.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index 5c112cac7..b7d52d497 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -108,6 +108,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 534562cca..460e9d4a2 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2038,3 +2038,11 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
{
return 0;
}
+
+__rte_weak uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+ struct rte_mbuf **tx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 8a7b459eb..43cee4244 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -23,6 +23,24 @@
#define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
FLAGS_BITS_OFFSET)
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+/* id bits offset in packed ring desc higher 64bits */
+#define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+/* net hdr short size mask */
+#define NET_HDR_MASK 0x3F
+
#define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
sizeof(struct vring_packed_desc))
#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
@@ -60,6 +78,237 @@ virtio_update_batch_stats(struct virtnet_stats *stats,
stats->bytes += pkt_len4;
}
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+ struct rte_mbuf **tx_pkts)
+{
+ struct virtqueue *vq = txvq->vq;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
+ uint16_t idx = vq->vq_avail_idx;
+ struct virtio_net_hdr *hdr;
+ uint16_t i, cmp;
+
+ if (vq->vq_avail_idx & PACKED_BATCH_MASK)
+ return -1;
+
+ if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+ return -1;
+
+ /* Load four mbufs rearm data */
+ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+ RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
+ __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
+ *tx_pkts[2]->rearm_data,
+ *tx_pkts[1]->rearm_data,
+ *tx_pkts[0]->rearm_data);
+
+ /* refcnt=1 and nb_segs=1 */
+ __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
+ __m256i head_rooms = _mm256_set1_epi16(head_size);
+
+ /* Check refcnt and nb_segs */
+ const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
+ cmp = _mm256_mask_cmpneq_epu16_mask(mask, mbufs, mbuf_ref);
+ if (unlikely(cmp))
+ return -1;
+
+ /* Check headroom is enough */
+ const __mmask16 data_mask = 0x1 | 0x1 << 4 | 0x1 << 8 | 0x1 << 12;
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
+ offsetof(struct rte_mbuf, rearm_data));
+ cmp = _mm256_mask_cmplt_epu16_mask(data_mask, mbufs, head_rooms);
+ if (unlikely(cmp))
+ return -1;
+
+ __m512i v_descx = _mm512_set_epi64(0x1, (uint64_t)tx_pkts[3],
+ 0x1, (uint64_t)tx_pkts[2],
+ 0x1, (uint64_t)tx_pkts[1],
+ 0x1, (uint64_t)tx_pkts[0]);
+
+ _mm512_storeu_si512((void *)&vq->vq_descx[idx], v_descx);
+
+ virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
+ tx_pkts[i]->data_off -= head_size;
+ tx_pkts[i]->d
[dpdk-dev] [PATCH v10 3/9] net/virtio: add vectorized devarg
Previously, virtio split ring vectorized path was enabled by default.
This is not suitable for everyone because that path dose not follow
virtio spec. Add new devarg for virtio vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 6286286db..902a1f0cf 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -363,6 +363,13 @@ Below devargs are supported by the PCI virtio driver:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfer to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
+
Below devargs are supported by the virtio-user vdev:
#. ``path``:
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 37766cbb6..0a69a4db1 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -48,7 +48,8 @@ static int virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev);
static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
- uint32_t *speed);
+ uint32_t *speed,
+ int *vectorized);
static int virtio_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info);
static int virtio_dev_link_update(struct rte_eth_dev *dev,
@@ -1551,8 +1552,8 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
}
} else {
- if (hw->use_simple_rx) {
- PMD_INIT_LOG(INFO, "virtio: using simple Rx path on
port %u",
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on
port %u",
eth_dev->data->port_id);
eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
} else if (hw->use_inorder_rx) {
@@ -1886,6 +1887,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
{
struct virtio_hw *hw = eth_dev->data->dev_private;
uint32_t speed = SPEED_UNKNOWN;
+ int vectorized = 0;
int ret;
if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
@@ -1912,7 +1914,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
ret = virtio_dev_devargs_parse(eth_dev->device->devargs,
-NULL, &speed);
+NULL, &speed, &vectorized);
if (ret < 0)
return ret;
hw->speed = speed;
@@ -1949,6 +1951,11 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
if (ret < 0)
goto err_virtio_init;
+ if (vectorized) {
+ if (!vtpci_packed_queue(hw))
+ hw->use_vec_rx = 1;
+ }
+
hw->opened = true;
return 0;
@@ -2021,9 +2028,20 @@ virtio_dev_speed_capa_get(uint32_t speed)
}
}
+static int vectorized_check_handler(__rte_unused const char *key,
+ const char *value, void *ret_val)
+{
+ if (strcmp(value, "1") == 0)
+ *(int *)ret_val = 1;
+ else
+ *(int *)ret_val = 0;
+
+ return 0;
+}
#define VIRTIO_ARG_SPEED "speed"
#define VIRTIO_ARG_VDPA "vdpa"
+#define VIRTIO_ARG_VECTORIZED "vectorized"
static int
@@ -2045,7 +2063,7 @@ link_speed_handler(const char *key __rte_unused,
static int
virtio_dev_devargs_parse(struct rte_devargs *devargs, int *vdpa,
- uint32_t *speed)
+ uint32_t *speed, int *vectorized)
{
struct rte_kvargs *kvlist;
int ret = 0;
@@ -2081,6 +2099,18 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
}
}
+ if (vectorized &&
+ rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
+ if (ret < 0) {
+ PMD_INIT_LOG(ERR, "Failed to parse %s",
+ VIRTIO_ARG_VECTORIZED);
+ goto exit;
+ }
+ }
+
exit:
rte_kvargs_free(kvlist);
return ret;
@@ -2092,7 +2122,8 @@ static int eth_virtio_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused,
int vdpa = 0;
int ret = 0;
- ret = virtio_dev_devargs_parse(pci_dev->device.devargs, &vdpa, NULL);
+ ret = virtio_dev_dev
[dpdk-dev] [PATCH v9 0/9] add packed ring vectorized path
This patch set introduced vectorized path for packed ring. The size of packed ring descriptor is 16Bytes. Four batched descriptors are just placed into one cacheline. AVX512 instructions can well handle this kind of data. Packed ring TX path can fully transformed into vectorized path. Packed ring Rx path can be vectorized when requirements met(LRO and mergeable disabled). New option RTE_LIBRTE_VIRTIO_INC_VECTOR will be introduced in this patch set. This option will unify split and packed ring vectorized path default setting. Meanwhile user can specify whether enable vectorized path at runtime by 'vectorized' parameter of virtio user vdev. v10: * reuse packed ring xmit cleanup v9: * replace RTE_LIBRTE_VIRTIO_INC_VECTOR with vectorized devarg * reorder patch sequence v8: * fix meson build error on ubuntu16.04 and suse15 v7: * default vectorization is disabled * compilation time check dependency on rte_mbuf structure * offsets are calcuated when compiling * remove useless barrier as descs are batched store&load * vindex of scatter is directly set * some comments updates * enable vectorized path in meson build v6: * fix issue when size not power of 2 v5: * remove cpuflags definition as required extensions always come with AVX512F on x86_64 * inorder actions should depend on feature bit * check ring type in rx queue setup * rewrite some commit logs * fix some checkpatch warnings v4: * rename 'packed_vec' to 'vectorized', also used in split ring * add RTE_LIBRTE_VIRTIO_INC_VECTOR config for virtio ethdev * check required AVX512 extensions cpuflags * combine split and packed ring datapath selection logic * remove limitation that size must power of two * clear 12Bytes virtio_net_hdr v3: * remove virtio_net_hdr array for better performance * disable 'packed_vec' by default v2: * more function blocks replaced by vector instructions * clean virtio_net_hdr by vector instruction * allow header room size change * add 'packed_vec' option in virtio_user vdev * fix build not check whether AVX512 enabled * doc update Tested-by: Wang, Yinan Marvin Liu (9): net/virtio: add Rx free threshold setting net/virtio: inorder should depend on feature bit net/virtio: add vectorized devarg net/virtio-user: add vectorized devarg net/virtio: reuse packed ring functions net/virtio: add vectorized packed ring Rx path net/virtio: add vectorized packed ring Tx path net/virtio: add election for vectorized path doc: add packed vectorized path doc/guides/nics/virtio.rst | 52 +- drivers/net/virtio/Makefile | 35 ++ drivers/net/virtio/meson.build | 14 + drivers/net/virtio/virtio_ethdev.c | 137 - drivers/net/virtio/virtio_ethdev.h | 6 + drivers/net/virtio/virtio_pci.h | 3 +- drivers/net/virtio/virtio_rxtx.c| 349 ++- drivers/net/virtio/virtio_rxtx_packed_avx.c | 623 drivers/net/virtio/virtio_user_ethdev.c | 32 +- drivers/net/virtio/virtqueue.c | 7 +- drivers/net/virtio/virtqueue.h | 307 +- 11 files changed, 1210 insertions(+), 355 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c -- 2.17.1
[dpdk-dev] [PATCH v10 1/9] net/virtio: add Rx free threshold setting
Introduce free threshold setting in Rx queue, its default value is 32.
Limit the threshold size to multiple of four as only vectorized packed
Rx function will utilize it. Virtio driver will rearm Rx queue when
more than rx_free_thresh descs were dequeued.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 060410577..94ba7a3ec 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -936,6 +936,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
struct virtio_hw *hw = dev->data->dev_private;
struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
struct virtnet_rx *rxvq;
+ uint16_t rx_free_thresh;
PMD_INIT_FUNC_TRACE();
@@ -944,6 +945,28 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
return -EINVAL;
}
+ rx_free_thresh = rx_conf->rx_free_thresh;
+ if (rx_free_thresh == 0)
+ rx_free_thresh =
+ RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
+
+ if (rx_free_thresh & 0x3) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+
+ if (rx_free_thresh >= vq->vq_nentries) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the "
+ "number of RX entries (%u)."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ vq->vq_nentries,
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+ vq->vq_free_thresh = rx_free_thresh;
+
if (nb_desc == 0 || nb_desc > vq->vq_nentries)
nb_desc = vq->vq_nentries;
vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 58ad7309a..6301c56b2 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -18,6 +18,8 @@
struct rte_mbuf;
+#define DEFAULT_RX_FREE_THRESH 32
+
/*
* Per virtio_ring.h in Linux.
* For virtio_pci on SMP, we don't need to order with respect to MMIO
--
2.17.1
[dpdk-dev] [PATCH v10 2/9] net/virtio: inorder should depend on feature bit
Ring initialization is different when inorder feature negotiated. This
action should dependent on negotiated feature bits.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 94ba7a3ec..e450477e8 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -989,6 +989,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
struct rte_mbuf *m;
uint16_t desc_idx;
int error, nbufs, i;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
PMD_INIT_FUNC_TRACE();
@@ -1018,7 +1019,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
virtio_rxq_rearm_vec(rxvq);
nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
}
- } else if (hw->use_inorder_rx) {
+ } else if (!vtpci_packed_queue(vq->hw) && in_order) {
if ((!virtqueue_full(vq))) {
uint16_t free_cnt = vq->vq_free_cnt;
struct rte_mbuf *pkts[free_cnt];
@@ -1133,7 +1134,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
PMD_INIT_FUNC_TRACE();
if (!vtpci_packed_queue(hw)) {
- if (hw->use_inorder_tx)
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER))
vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
}
@@ -2046,7 +2047,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_tx = 0;
- bool in_order = hw->use_inorder_tx;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
return nb_tx;
--
2.17.1
[dpdk-dev] [PATCH v10 4/9] net/virtio-user: add vectorized devarg
Add new devarg for virtio user device vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 902a1f0cf..d59add23e 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -424,6 +424,12 @@ Below devargs are supported by the virtio-user vdev:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfer to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
Virtio paths Selection and Usage
diff --git a/drivers/net/virtio/virtio_user_ethdev.c
b/drivers/net/virtio/virtio_user_ethdev.c
index 150a8d987..40ad786cc 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -452,6 +452,8 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_PACKED_VQ,
#define VIRTIO_USER_ARG_SPEED "speed"
VIRTIO_USER_ARG_SPEED,
+#define VIRTIO_USER_ARG_VECTORIZED "vectorized"
+ VIRTIO_USER_ARG_VECTORIZED,
NULL
};
@@ -559,6 +561,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t mrg_rxbuf = 1;
uint64_t in_order = 1;
uint64_t packed_vq = 0;
+ uint64_t vectorized = 0;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
@@ -675,6 +678,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
+ &get_integer_arg, &vectorized) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+VIRTIO_USER_ARG_VECTORIZED);
+ goto end;
+ }
+ }
+
if (queues > 1 && cq == 0) {
PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
goto end;
@@ -727,6 +739,9 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vectorized)
+ hw->use_vec_rx = 1;
+
rte_eth_dev_probing_finish(eth_dev);
ret = 0;
@@ -785,4 +800,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
"mrg_rxbuf=<0|1> "
"in_order=<0|1> "
"packed_vq=<0|1> "
- "speed=");
+ "speed= "
+ "vectorized=<0|1>");
--
2.17.1
[dpdk-dev] [PATCH v9 9/9] doc: add packed vectorized path
Document packed virtqueue vectorized path selection logic in virtio net PMD. Signed-off-by: Marvin Liu diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index d59add23e..dbcf49ae1 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -482,6 +482,13 @@ according to below configuration: both negotiated, this path will be selected. #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. +#. Packed virtqueue vectorized Rx path: If building and running environment support + AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && + TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. +#. Packed virtqueue vectorized Tx path: If building and running environment support + AVX512 && in-order feature is negotiated && vectorized option enabled, + this path will be selected. Rx/Tx callbacks of each Virtio path ~~~ @@ -504,6 +511,8 @@ are shown in below table: Packed virtqueue non-meregable path virtio_recv_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order mergeable path virtio_recv_mergeable_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order non-mergeable path virtio_recv_pkts_packed virtio_xmit_pkts_packed + Packed virtqueue vectorized Rx path virtio_recv_pkts_packed_vec virtio_xmit_pkts_packed + Packed virtqueue vectorized Tx path virtio_recv_pkts_packed virtio_xmit_pkts_packed_vec = Virtio paths Support Status from Release to Release @@ -521,20 +530,22 @@ All virtio paths support status are shown in below table: .. table:: Virtio Paths and Releases - = = = - Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 - = = = - Split virtqueue mergeable path Y Y Y - Split virtqueue non-mergeable path Y Y Y - Split virtqueue vectorized Rx path Y Y Y - Split virtqueue simple Tx path Y N N - Split virtqueue in-order mergeable path Y Y - Split virtqueue in-order non-mergeable path Y Y - Packed virtqueue mergeable path Y - Packed virtqueue non-mergeable path Y - Packed virtqueue in-order mergeable path Y - Packed virtqueue in-order non-mergeable path Y - = = = + = = = === + Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 20.05 ~ + = = = === + Split virtqueue mergeable path Y Y Y Y + Split virtqueue non-mergeable path Y Y Y Y + Split virtqueue vectorized Rx path Y Y Y Y + Split virtqueue simple Tx path Y N N N + Split virtqueue in-order mergeable path Y Y Y + Split virtqueue in-order non-mergeable path Y Y Y + Packed virtqueue mergeable path Y Y + Packed virtqueue non-mergeable path Y Y + Packed virtqueue in-order mergeable path Y Y + Packed virtqueue in-order non-mergeable path Y Y + Packed virtqueue vectorized Rx path Y + Packed virtqueue vectorized Tx path Y + = = = === QEMU Support Status ~~~ -- 2.17.1
[dpdk-dev] [PATCH v9 8/9] net/virtio: add election for vectorized path
Rewrite vectorized path selection logic. Default setting comes from
vectorized devarg, then checks each criteria.
Packed ring vectorized path need:
AVX512F and required extensions are supported by compiler and host
VERSION_1 and IN_ORDER features are negotiated
mergeable feature is not negotiated
LRO offloading is disabled
Split ring vectorized rx path need:
mergeable and IN_ORDER features are not negotiated
LRO, chksum and vlan strip offloadings are disabled
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 0a69a4db1..8a9545dd8 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1523,9 +1523,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
if (vtpci_packed_queue(hw)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring %s Tx path on port %u",
- hw->use_inorder_tx ? "inorder" : "standard",
+ hw->use_vec_tx ? "vectorized" : "standard",
eth_dev->data->port_id);
- eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
+ if (hw->use_vec_tx)
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
+ else
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
} else {
if (hw->use_inorder_tx) {
PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on
port %u",
@@ -1539,7 +1542,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
}
if (vtpci_packed_queue(hw)) {
- if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO,
+ "virtio: using packed ring vectorized Rx path
on port %u",
+ eth_dev->data->port_id);
+ eth_dev->rx_pkt_burst =
+ &virtio_recv_pkts_packed_vec;
+ } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring mergeable buffer Rx
path on port %u",
eth_dev->data->port_id);
@@ -1952,8 +1961,17 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
goto err_virtio_init;
if (vectorized) {
- if (!vtpci_packed_queue(hw))
+ if (!vtpci_packed_queue(hw)) {
+ hw->use_vec_rx = 1;
+ } else {
+#if !defined(CC_AVX512_SUPPORT)
+ PMD_DRV_LOG(INFO,
+ "building environment do not support packed
ring vectorized");
+#else
hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
+#endif
+ }
}
hw->opened = true;
@@ -2099,11 +2117,10 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
}
}
- if (vectorized &&
- rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
+ if (vectorized && rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1)
{
ret = rte_kvargs_process(kvlist,
- VIRTIO_ARG_VECTORIZED,
- vectorized_check_handler, vectorized);
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
if (ret < 0) {
PMD_INIT_LOG(ERR, "Failed to parse %s",
VIRTIO_ARG_VECTORIZED);
@@ -2288,31 +2305,61 @@ virtio_dev_configure(struct rte_eth_dev *dev)
return -EBUSY;
}
- if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
- hw->use_inorder_tx = 1;
- hw->use_inorder_rx = 1;
- hw->use_vec_rx = 0;
- }
-
if (vtpci_packed_queue(hw)) {
- hw->use_vec_rx = 0;
- hw->use_inorder_rx = 0;
- }
+ if ((hw->use_vec_rx || hw->use_vec_tx) &&
+ (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
+!vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+!vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized path for
requirements not met");
+ hw->use_vec_rx = 0;
+ hw->use_vec_tx = 0;
+ }
+ if (hw->use_vec_rx)
[dpdk-dev] [PATCH v9 3/9] net/virtio: add vectorized devarg
Previously, virtio split ring vectorized path was enabled by default.
This is not suitable for everyone because that path dose not follow
virtio spec. Add new devarg for virtio vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 6286286db..902a1f0cf 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -363,6 +363,13 @@ Below devargs are supported by the PCI virtio driver:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfer to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
+
Below devargs are supported by the virtio-user vdev:
#. ``path``:
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 37766cbb6..0a69a4db1 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -48,7 +48,8 @@ static int virtio_dev_allmulticast_disable(struct rte_eth_dev
*dev);
static uint32_t virtio_dev_speed_capa_get(uint32_t speed);
static int virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
- uint32_t *speed);
+ uint32_t *speed,
+ int *vectorized);
static int virtio_dev_info_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info);
static int virtio_dev_link_update(struct rte_eth_dev *dev,
@@ -1551,8 +1552,8 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed;
}
} else {
- if (hw->use_simple_rx) {
- PMD_INIT_LOG(INFO, "virtio: using simple Rx path on
port %u",
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on
port %u",
eth_dev->data->port_id);
eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
} else if (hw->use_inorder_rx) {
@@ -1886,6 +1887,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
{
struct virtio_hw *hw = eth_dev->data->dev_private;
uint32_t speed = SPEED_UNKNOWN;
+ int vectorized = 0;
int ret;
if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) {
@@ -1912,7 +1914,7 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
return 0;
}
ret = virtio_dev_devargs_parse(eth_dev->device->devargs,
-NULL, &speed);
+NULL, &speed, &vectorized);
if (ret < 0)
return ret;
hw->speed = speed;
@@ -1949,6 +1951,11 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
if (ret < 0)
goto err_virtio_init;
+ if (vectorized) {
+ if (!vtpci_packed_queue(hw))
+ hw->use_vec_rx = 1;
+ }
+
hw->opened = true;
return 0;
@@ -2021,9 +2028,20 @@ virtio_dev_speed_capa_get(uint32_t speed)
}
}
+static int vectorized_check_handler(__rte_unused const char *key,
+ const char *value, void *ret_val)
+{
+ if (strcmp(value, "1") == 0)
+ *(int *)ret_val = 1;
+ else
+ *(int *)ret_val = 0;
+
+ return 0;
+}
#define VIRTIO_ARG_SPEED "speed"
#define VIRTIO_ARG_VDPA "vdpa"
+#define VIRTIO_ARG_VECTORIZED "vectorized"
static int
@@ -2045,7 +2063,7 @@ link_speed_handler(const char *key __rte_unused,
static int
virtio_dev_devargs_parse(struct rte_devargs *devargs, int *vdpa,
- uint32_t *speed)
+ uint32_t *speed, int *vectorized)
{
struct rte_kvargs *kvlist;
int ret = 0;
@@ -2081,6 +2099,18 @@ virtio_dev_devargs_parse(struct rte_devargs *devargs,
int *vdpa,
}
}
+ if (vectorized &&
+ rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) {
+ ret = rte_kvargs_process(kvlist,
+ VIRTIO_ARG_VECTORIZED,
+ vectorized_check_handler, vectorized);
+ if (ret < 0) {
+ PMD_INIT_LOG(ERR, "Failed to parse %s",
+ VIRTIO_ARG_VECTORIZED);
+ goto exit;
+ }
+ }
+
exit:
rte_kvargs_free(kvlist);
return ret;
@@ -2092,7 +2122,8 @@ static int eth_virtio_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused,
int vdpa = 0;
int ret = 0;
- ret = virtio_dev_devargs_parse(pci_dev->device.devargs, &vdpa, NULL);
+ ret = virtio_dev_devargs_parse(pci_dev->device.
[dpdk-dev] [PATCH v9 7/9] net/virtio: add vectorized packed ring Tx path
Optimize packed ring Tx path alike Rx path. Split Tx path into batch and
single Tx functions. Batch function is further optimized by AVX512
instructions.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index 5c112cac7..b7d52d497 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -108,6 +108,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index cf18fe564..f82fe8d64 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2175,3 +2175,11 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
{
return 0;
}
+
+__rte_weak uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+ struct rte_mbuf **tx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 8a7b459eb..c023ace4e 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -23,6 +23,24 @@
#define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
FLAGS_BITS_OFFSET)
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+/* id bits offset in packed ring desc higher 64bits */
+#define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+/* net hdr short size mask */
+#define NET_HDR_MASK 0x3F
+
#define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
sizeof(struct vring_packed_desc))
#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
@@ -47,6 +65,48 @@
for (iter = val; iter < num; iter++)
#endif
+static inline void
+virtio_xmit_cleanup_packed_vec(struct virtqueue *vq)
+{
+ struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
+ struct vq_desc_extra *dxp;
+ uint16_t used_idx, id, curr_id, free_cnt = 0;
+ uint16_t size = vq->vq_nentries;
+ struct rte_mbuf *mbufs[size];
+ uint16_t nb_mbuf = 0, i;
+
+ used_idx = vq->vq_used_cons_idx;
+
+ if (!desc_is_used(&desc[used_idx], vq))
+ return;
+
+ id = desc[used_idx].id;
+
+ do {
+ curr_id = used_idx;
+ dxp = &vq->vq_descx[used_idx];
+ used_idx += dxp->ndescs;
+ free_cnt += dxp->ndescs;
+
+ if (dxp->cookie != NULL) {
+ mbufs[nb_mbuf] = dxp->cookie;
+ dxp->cookie = NULL;
+ nb_mbuf++;
+ }
+
+ if (used_idx >= size) {
+ used_idx -= size;
+ vq->vq_packed.used_wrap_counter ^= 1;
+ }
+ } while (curr_id != id);
+
+ for (i = 0; i < nb_mbuf; i++)
+ rte_pktmbuf_free(mbufs[i]);
+
+ vq->vq_used_cons_idx = used_idx;
+ vq->vq_free_cnt += free_cnt;
+}
+
static inline void
virtio_update_batch_stats(struct virtnet_stats *stats,
uint16_t pkt_len1,
@@ -60,6 +120,237 @@ virtio_update_batch_stats(struct virtnet_stats *stats,
stats->bytes += pkt_len4;
}
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+ struct rte_mbuf **tx_pkts)
+{
+ struct virtqueue *vq = txvq->vq;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
+ uint16_t idx = vq->vq_avail_idx;
+ struct virtio_net_hdr *hdr;
+ uint16_t i, cmp;
+
+ if (vq->vq_avail_idx & PACKED_BATCH_MASK)
+ return -1;
+
+ if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+ return -1;
+
+ /* Load four mbufs rearm data */
+ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+ RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
+ __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
+
[dpdk-dev] [PATCH v9 4/9] net/virtio-user: add vectorized devarg
Add new devarg for virtio user device vectorized path selection. By
default vectorized path is disabled.
Signed-off-by: Marvin Liu
diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst
index 902a1f0cf..d59add23e 100644
--- a/doc/guides/nics/virtio.rst
+++ b/doc/guides/nics/virtio.rst
@@ -424,6 +424,12 @@ Below devargs are supported by the virtio-user vdev:
rte_eth_link_get_nowait function.
(Default: 1 (10G))
+#. ``vectorized``:
+
+It is used to specify whether virtio device perfer to use vectorized path.
+Afterwards, dependencies of vectorized path will be checked in path
+election.
+(Default: 0 (disabled))
Virtio paths Selection and Usage
diff --git a/drivers/net/virtio/virtio_user_ethdev.c
b/drivers/net/virtio/virtio_user_ethdev.c
index 150a8d987..40ad786cc 100644
--- a/drivers/net/virtio/virtio_user_ethdev.c
+++ b/drivers/net/virtio/virtio_user_ethdev.c
@@ -452,6 +452,8 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_PACKED_VQ,
#define VIRTIO_USER_ARG_SPEED "speed"
VIRTIO_USER_ARG_SPEED,
+#define VIRTIO_USER_ARG_VECTORIZED "vectorized"
+ VIRTIO_USER_ARG_VECTORIZED,
NULL
};
@@ -559,6 +561,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t mrg_rxbuf = 1;
uint64_t in_order = 1;
uint64_t packed_vq = 0;
+ uint64_t vectorized = 0;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
@@ -675,6 +678,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
+ if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
+ if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
+ &get_integer_arg, &vectorized) < 0) {
+ PMD_INIT_LOG(ERR, "error to parse %s",
+VIRTIO_USER_ARG_VECTORIZED);
+ goto end;
+ }
+ }
+
if (queues > 1 && cq == 0) {
PMD_INIT_LOG(ERR, "multi-q requires ctrl-q");
goto end;
@@ -727,6 +739,9 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
goto end;
}
+ if (vectorized)
+ hw->use_vec_rx = 1;
+
rte_eth_dev_probing_finish(eth_dev);
ret = 0;
@@ -785,4 +800,5 @@ RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
"mrg_rxbuf=<0|1> "
"in_order=<0|1> "
"packed_vq=<0|1> "
- "speed=");
+ "speed= "
+ "vectorized=<0|1>");
--
2.17.1
[dpdk-dev] [PATCH v9 6/9] net/virtio: reuse packed ring xmit functions
Move xmit offload and packed ring xmit enqueue function to header file.
These functions will be reused by packed ring vectorized Tx function.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index c9b6e7844..cf18fe564 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -264,10 +264,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
return i;
}
-#ifndef DEFAULT_TX_FREE_THRESH
-#define DEFAULT_TX_FREE_THRESH 32
-#endif
-
static void
virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
{
@@ -562,68 +558,7 @@ virtio_tso_fix_cksum(struct rte_mbuf *m)
}
-/* avoid write operation when necessary, to lessen cache issues */
-#define ASSIGN_UNLESS_EQUAL(var, val) do { \
- if ((var) != (val)) \
- (var) = (val); \
-} while (0)
-
-#define virtqueue_clear_net_hdr(_hdr) do { \
- ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0);\
- ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0);\
-} while (0)
-
-static inline void
-virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
- struct rte_mbuf *cookie,
- bool offload)
-{
- if (offload) {
- if (cookie->ol_flags & PKT_TX_TCP_SEG)
- cookie->ol_flags |= PKT_TX_TCP_CKSUM;
-
- switch (cookie->ol_flags & PKT_TX_L4_MASK) {
- case PKT_TX_UDP_CKSUM:
- hdr->csum_start = cookie->l2_len + cookie->l3_len;
- hdr->csum_offset = offsetof(struct rte_udp_hdr,
- dgram_cksum);
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- break;
-
- case PKT_TX_TCP_CKSUM:
- hdr->csum_start = cookie->l2_len + cookie->l3_len;
- hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum);
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- break;
-
- default:
- ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
- ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
- ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
- break;
- }
- /* TCP Segmentation Offload */
- if (cookie->ol_flags & PKT_TX_TCP_SEG) {
- hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
- VIRTIO_NET_HDR_GSO_TCPV6 :
- VIRTIO_NET_HDR_GSO_TCPV4;
- hdr->gso_size = cookie->tso_segsz;
- hdr->hdr_len =
- cookie->l2_len +
- cookie->l3_len +
- cookie->l4_len;
- } else {
- ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
- ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
- ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
- }
- }
-}
static inline void
virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
@@ -725,102 +660,6 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx
*txvq,
virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
}
-static inline void
-virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
- uint16_t needed, int can_push, int in_order)
-{
- struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
- struct vq_desc_extra *dxp;
- struct virtqueue *vq = txvq->vq;
- struct vring_packed_desc *start_dp, *head_dp;
- uint16_t idx, id, head_idx, head_flags;
- int16_t head_size = vq->hw->vtnet_hdr_size;
- struct virtio_net_hdr *hdr;
- uint16_t prev;
- bool prepend_header = false;
-
- id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
-
- dxp = &vq->vq_descx[id];
- dxp->ndescs = needed;
- dxp->cookie = cookie;
-
- head_idx = vq->vq_avail_idx;
- idx = head_idx;
- prev = head_idx;
- start_dp = vq->vq_packed.ring.desc;
-
- head_dp = &vq->vq_packed.ring.desc[idx];
- head_flags = cookie->next ? VRING_DESC_F_NEXT : 0;
- head_flags |= vq->vq_packed.cached_flags;
-
- if (can_push) {
- /* prepend cannot fail, checked by caller */
- hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr
[dpdk-dev] [PATCH v9 5/9] net/virtio: add vectorized packed ring Rx path
Optimize packed ring Rx path with SIMD instructions. Solution of
optimization is pretty like vhost, is that split path into batch and
single functions. Batch function is further optimized by AVX512
instructions. Also pad desc extra structure to 16 bytes aligned, thus
four elements will be saved in one batch.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index c9edb84ee..102b1deab 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -36,6 +36,41 @@ else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM)
$(CONFIG_RTE_ARCH_ARM64)),)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c
endif
+ifneq ($(FORCE_DISABLE_AVX512), y)
+ CC_AVX512_SUPPORT=\
+ $(shell $(CC) -march=native -dM -E - &1 | \
+ sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+ grep -q AVX512 && echo 1)
+endif
+
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_packed_avx.c
+
+ifeq ($(RTE_TOOLCHAIN), gcc)
+ifeq ($(shell test $(GCC_VERSION) -ge 83 && echo 1), 1)
+CFLAGS += -DVIRTIO_GCC_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), clang)
+ifeq ($(shell test $(CLANG_MAJOR_VERSION)$(CLANG_MINOR_VERSION) -ge 37 && echo
1), 1)
+CFLAGS += -DVIRTIO_CLANG_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), icc)
+ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 16 && echo 1), 1)
+CFLAGS += -DVIRTIO_ICC_UNROLL_PRAGMA
+endif
+endif
+
+CFLAGS_virtio_rxtx_packed_avx.o += -mavx512f -mavx512bw -mavx512vl
+ifeq ($(shell test $(GCC_VERSION) -ge 100 && echo 1), 1)
+CFLAGS_virtio_rxtx_packed_avx.o += -Wno-zero-length-bounds
+endif
+endif
+
ifeq ($(CONFIG_RTE_VIRTIO_USER),y)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_user.c
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_user/vhost_kernel.c
diff --git a/drivers/net/virtio/meson.build b/drivers/net/virtio/meson.build
index 15150eea1..8e68c3039 100644
--- a/drivers/net/virtio/meson.build
+++ b/drivers/net/virtio/meson.build
@@ -9,6 +9,20 @@ sources += files('virtio_ethdev.c',
deps += ['kvargs', 'bus_pci']
if arch_subdir == 'x86'
+ if '-mno-avx512f' not in machine_args
+ if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+ cflags += ['-mavx512f', '-mavx512bw', '-mavx512vl']
+ cflags += ['-DCC_AVX512_SUPPORT']
+ if (toolchain == 'gcc' and
cc.version().version_compare('>=8.3.0'))
+ cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
+ elif (toolchain == 'clang' and
cc.version().version_compare('>=3.7.0'))
+ cflags += '-DVHOST_CLANG_UNROLL_PRAGMA'
+ elif (toolchain == 'icc' and
cc.version().version_compare('>=16.0.0'))
+ cflags += '-DVHOST_ICC_UNROLL_PRAGMA'
+ endif
+ sources += files('virtio_rxtx_packed_avx.c')
+ endif
+ endif
sources += files('virtio_rxtx_simple_sse.c')
elif arch_subdir == 'ppc'
sources += files('virtio_rxtx_simple_altivec.c')
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index febaf17a8..5c112cac7 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -105,6 +105,9 @@ uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct
rte_mbuf **tx_pkts,
uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 84f4cf946..c9b6e7844 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2329,3 +2329,11 @@ virtio_xmit_pkts_inorder(void *tx_queue,
return nb_tx;
}
+
+__rte_weak uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+ struct rte_mbuf **rx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
new file mode 100644
index 0..8a7b459eb
--- /dev/null
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -0,0 +1,374 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+
[dpdk-dev] [PATCH v9 2/9] net/virtio: inorder should depend on feature bit
Ring initialization is different when inorder feature negotiated. This
action should dependent on negotiated feature bits.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 94ba7a3ec..e450477e8 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -989,6 +989,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
struct rte_mbuf *m;
uint16_t desc_idx;
int error, nbufs, i;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
PMD_INIT_FUNC_TRACE();
@@ -1018,7 +1019,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
virtio_rxq_rearm_vec(rxvq);
nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
}
- } else if (hw->use_inorder_rx) {
+ } else if (!vtpci_packed_queue(vq->hw) && in_order) {
if ((!virtqueue_full(vq))) {
uint16_t free_cnt = vq->vq_free_cnt;
struct rte_mbuf *pkts[free_cnt];
@@ -1133,7 +1134,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
PMD_INIT_FUNC_TRACE();
if (!vtpci_packed_queue(hw)) {
- if (hw->use_inorder_tx)
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER))
vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
}
@@ -2046,7 +2047,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_tx = 0;
- bool in_order = hw->use_inorder_tx;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
return nb_tx;
--
2.17.1
[dpdk-dev] [PATCH v9 1/9] net/virtio: add Rx free threshold setting
Introduce free threshold setting in Rx queue, its default value is 32.
Limit the threshold size to multiple of four as only vectorized packed
Rx function will utilize it. Virtio driver will rearm Rx queue when
more than rx_free_thresh descs were dequeued.
Signed-off-by: Marvin Liu
Reviewed-by: Maxime Coquelin
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 060410577..94ba7a3ec 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -936,6 +936,7 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
struct virtio_hw *hw = dev->data->dev_private;
struct virtqueue *vq = hw->vqs[vtpci_queue_idx];
struct virtnet_rx *rxvq;
+ uint16_t rx_free_thresh;
PMD_INIT_FUNC_TRACE();
@@ -944,6 +945,28 @@ virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
return -EINVAL;
}
+ rx_free_thresh = rx_conf->rx_free_thresh;
+ if (rx_free_thresh == 0)
+ rx_free_thresh =
+ RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
+
+ if (rx_free_thresh & 0x3) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be multiples of four."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+
+ if (rx_free_thresh >= vq->vq_nentries) {
+ RTE_LOG(ERR, PMD, "rx_free_thresh must be less than the "
+ "number of RX entries (%u)."
+ " (rx_free_thresh=%u port=%u queue=%u)\n",
+ vq->vq_nentries,
+ rx_free_thresh, dev->data->port_id, queue_idx);
+ return -EINVAL;
+ }
+ vq->vq_free_thresh = rx_free_thresh;
+
if (nb_desc == 0 || nb_desc > vq->vq_nentries)
nb_desc = vq->vq_nentries;
vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
diff --git a/drivers/net/virtio/virtqueue.h b/drivers/net/virtio/virtqueue.h
index 58ad7309a..6301c56b2 100644
--- a/drivers/net/virtio/virtqueue.h
+++ b/drivers/net/virtio/virtqueue.h
@@ -18,6 +18,8 @@
struct rte_mbuf;
+#define DEFAULT_RX_FREE_THRESH 32
+
/*
* Per virtio_ring.h in Linux.
* For virtio_pci on SMP, we don't need to order with respect to MMIO
--
2.17.1
[dpdk-dev] [PATCH v9 0/9] add packed ring vectorized path
This patch set introduced vectorized path for packed ring. The size of packed ring descriptor is 16Bytes. Four batched descriptors are just placed into one cacheline. AVX512 instructions can well handle this kind of data. Packed ring TX path can fully transformed into vectorized path. Packed ring Rx path can be vectorized when requirements met(LRO and mergeable disabled). New option RTE_LIBRTE_VIRTIO_INC_VECTOR will be introduced in this patch set. This option will unify split and packed ring vectorized path default setting. Meanwhile user can specify whether enable vectorized path at runtime by 'vectorized' parameter of virtio user vdev. v9: * replace RTE_LIBRTE_VIRTIO_INC_VECTOR with vectorized devarg * reorder patch sequence v8: * fix meson build error on ubuntu16.04 and suse15 v7: * default vectorization is disabled * compilation time check dependency on rte_mbuf structure * offsets are calcuated when compiling * remove useless barrier as descs are batched store&load * vindex of scatter is directly set * some comments updates * enable vectorized path in meson build v6: * fix issue when size not power of 2 v5: * remove cpuflags definition as required extensions always come with AVX512F on x86_64 * inorder actions should depend on feature bit * check ring type in rx queue setup * rewrite some commit logs * fix some checkpatch warnings v4: * rename 'packed_vec' to 'vectorized', also used in split ring * add RTE_LIBRTE_VIRTIO_INC_VECTOR config for virtio ethdev * check required AVX512 extensions cpuflags * combine split and packed ring datapath selection logic * remove limitation that size must power of two * clear 12Bytes virtio_net_hdr v3: * remove virtio_net_hdr array for better performance * disable 'packed_vec' by default v2: * more function blocks replaced by vector instructions * clean virtio_net_hdr by vector instruction * allow header room size change * add 'packed_vec' option in virtio_user vdev * fix build not check whether AVX512 enabled * doc update Tested-by: Wang, Yinan Marvin Liu (9): net/virtio: add Rx free threshold setting net/virtio: inorder should depend on feature bit net/virtio: add vectorized devarg net/virtio-user: add vectorized devarg net/virtio: add vectorized packed ring Rx path net/virtio: reuse packed ring xmit functions net/virtio: add vectorized packed ring Tx path net/virtio: add election for vectorized path doc: add packed vectorized path doc/guides/nics/virtio.rst | 52 +- drivers/net/virtio/Makefile | 35 ++ drivers/net/virtio/meson.build | 14 + drivers/net/virtio/virtio_ethdev.c | 136 +++- drivers/net/virtio/virtio_ethdev.h | 6 + drivers/net/virtio/virtio_pci.h | 3 +- drivers/net/virtio/virtio_rxtx.c| 212 ++- drivers/net/virtio/virtio_rxtx_packed_avx.c | 665 drivers/net/virtio/virtio_user_ethdev.c | 32 +- drivers/net/virtio/virtqueue.c | 7 +- drivers/net/virtio/virtqueue.h | 168 - 11 files changed, 1112 insertions(+), 218 deletions(-) create mode 100644 drivers/net/virtio/virtio_rxtx_packed_avx.c -- 2.17.1
[dpdk-dev] [PATCH v8 8/9] net/virtio: add election for vectorized path
Rewrite vectorized path selection logic. Default setting comes from
RTE_LIBRTE_VIRTIO_INC_VECTOR option. Paths criteria will be checked as
listed below.
Packed ring vectorized path will be selected when:
vectorized option is enabled
AVX512F and required extensions are supported by compiler and host
virtio VERSION_1 and IN_ORDER features are negotiated
virtio mergeable feature is not negotiated
LRO offloading is disabled
Split ring vectorized rx path will be selected when:
vectorized option is enabled
virtio mergeable and IN_ORDER features are not negotiated
LRO, chksum and vlan strip offloading are disabled
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.c
b/drivers/net/virtio/virtio_ethdev.c
index 361c834a9..c700af6be 100644
--- a/drivers/net/virtio/virtio_ethdev.c
+++ b/drivers/net/virtio/virtio_ethdev.c
@@ -1522,9 +1522,12 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
if (vtpci_packed_queue(hw)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring %s Tx path on port %u",
- hw->use_inorder_tx ? "inorder" : "standard",
+ hw->use_vec_tx ? "vectorized" : "standard",
eth_dev->data->port_id);
- eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
+ if (hw->use_vec_tx)
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec;
+ else
+ eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed;
} else {
if (hw->use_inorder_tx) {
PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on
port %u",
@@ -1538,7 +1541,13 @@ set_rxtx_funcs(struct rte_eth_dev *eth_dev)
}
if (vtpci_packed_queue(hw)) {
- if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ if (hw->use_vec_rx) {
+ PMD_INIT_LOG(INFO,
+ "virtio: using packed ring vectorized Rx path
on port %u",
+ eth_dev->data->port_id);
+ eth_dev->rx_pkt_burst =
+ &virtio_recv_pkts_packed_vec;
+ } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
PMD_INIT_LOG(INFO,
"virtio: using packed ring mergeable buffer Rx
path on port %u",
eth_dev->data->port_id);
@@ -1950,6 +1959,10 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
goto err_virtio_init;
hw->opened = true;
+#ifdef RTE_LIBRTE_VIRTIO_INC_VECTOR
+ hw->use_vec_rx = 1;
+ hw->use_vec_tx = 1;
+#endif
return 0;
@@ -2257,33 +2270,63 @@ virtio_dev_configure(struct rte_eth_dev *dev)
return -EBUSY;
}
- hw->use_vec_rx = 1;
+ if (vtpci_packed_queue(hw)) {
+#if defined RTE_ARCH_X86
+ if ((hw->use_vec_rx || hw->use_vec_tx) &&
+ (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) ||
+!vtpci_with_feature(hw, VIRTIO_F_IN_ORDER) ||
+!vtpci_with_feature(hw, VIRTIO_F_VERSION_1))) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorization for
requirements are not met");
+ hw->use_vec_rx = 0;
+ hw->use_vec_tx = 0;
+ }
+#endif
- if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
- hw->use_inorder_tx = 1;
- hw->use_inorder_rx = 1;
- hw->use_vec_rx = 0;
- }
+ if (hw->use_vec_rx) {
+ if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
mrg_rxbuf enabled");
+ hw->use_vec_rx = 0;
+ }
- if (vtpci_packed_queue(hw)) {
- hw->use_vec_rx = 0;
- hw->use_inorder_rx = 0;
- }
+ if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
+ PMD_DRV_LOG(INFO,
+ "disabled packed ring vectorized rx for
TCP_LRO enabled");
+ hw->use_vec_rx = 0;
+ }
+ }
+ } else {
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER)) {
+ hw->use_inorder_tx = 1;
+ hw->use_inorder_rx = 1;
+ hw->use_vec_rx = 0;
+ }
+ if (hw->use_vec_rx
[dpdk-dev] [PATCH v8 9/9] doc: add packed vectorized path
Document packed virtqueue vectorized path selection logic in virtio net PMD. Signed-off-by: Marvin Liu diff --git a/doc/guides/nics/virtio.rst b/doc/guides/nics/virtio.rst index 6286286db..4bd46f83e 100644 --- a/doc/guides/nics/virtio.rst +++ b/doc/guides/nics/virtio.rst @@ -417,6 +417,10 @@ Below devargs are supported by the virtio-user vdev: rte_eth_link_get_nowait function. (Default: 1 (10G)) +#. ``vectorized``: + +It is used to enable virtio device vectorized path. +(Default: 0 (disabled)) Virtio paths Selection and Usage @@ -469,6 +473,13 @@ according to below configuration: both negotiated, this path will be selected. #. Packed virtqueue in-order non-mergeable path: If in-order feature is negotiated and Rx mergeable is not negotiated, this path will be selected. +#. Packed virtqueue vectorized Rx path: If building and running environment support + AVX512 && in-order feature is negotiated && Rx mergeable is not negotiated && + TCP_LRO Rx offloading is disabled && vectorized option enabled, + this path will be selected. +#. Packed virtqueue vectorized Tx path: If building and running environment support + AVX512 && in-order feature is negotiated && vectorized option enabled, + this path will be selected. Rx/Tx callbacks of each Virtio path ~~~ @@ -491,6 +502,8 @@ are shown in below table: Packed virtqueue non-meregable path virtio_recv_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order mergeable path virtio_recv_mergeable_pkts_packed virtio_xmit_pkts_packed Packed virtqueue in-order non-mergeable path virtio_recv_pkts_packed virtio_xmit_pkts_packed + Packed virtqueue vectorized Rx path virtio_recv_pkts_packed_vec virtio_xmit_pkts_packed + Packed virtqueue vectorized Tx path virtio_recv_pkts_packed virtio_xmit_pkts_packed_vec = Virtio paths Support Status from Release to Release @@ -508,20 +521,22 @@ All virtio paths support status are shown in below table: .. table:: Virtio Paths and Releases - = = = - Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 - = = = - Split virtqueue mergeable path Y Y Y - Split virtqueue non-mergeable path Y Y Y - Split virtqueue vectorized Rx path Y Y Y - Split virtqueue simple Tx path Y N N - Split virtqueue in-order mergeable path Y Y - Split virtqueue in-order non-mergeable path Y Y - Packed virtqueue mergeable path Y - Packed virtqueue non-mergeable path Y - Packed virtqueue in-order mergeable path Y - Packed virtqueue in-order non-mergeable path Y - = = = + = = = === + Virtio paths 16.11 ~ 18.05 18.08 ~ 18.11 19.02 ~ 19.11 20.05 ~ + = = = === + Split virtqueue mergeable path Y Y Y Y + Split virtqueue non-mergeable path Y Y Y Y + Split virtqueue vectorized Rx path Y Y Y Y + Split virtqueue simple Tx path Y N N N + Split virtqueue in-order mergeable path Y Y Y + Split virtqueue in-order non-mergeable path Y Y Y + Packed virtqueue mergeable path Y Y + Packed virtqueue non-mergeable path Y Y + Packed virtqueue in-order mergeable path Y Y + Packed virtqueue in-order non-mergeable path Y Y + Packed virtqueue vectorized Rx path Y + Packed virtqueue vectorized Tx path
[dpdk-dev] [PATCH v8 7/9] net/virtio: add vectorized packed ring Tx path
Optimize packed ring Tx path alike Rx path. Split Tx path into batch and
single Tx functions. Batch function is further optimized by vector
instructions.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index 5c112cac7..b7d52d497 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -108,6 +108,9 @@ uint16_t virtio_recv_pkts_vec(void *rx_queue, struct
rte_mbuf **rx_pkts,
uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index cf18fe564..f82fe8d64 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -2175,3 +2175,11 @@ virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
{
return 0;
}
+
+__rte_weak uint16_t
+virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
+ struct rte_mbuf **tx_pkts __rte_unused,
+ uint16_t nb_pkts __rte_unused)
+{
+ return 0;
+}
diff --git a/drivers/net/virtio/virtio_rxtx_packed_avx.c
b/drivers/net/virtio/virtio_rxtx_packed_avx.c
index 3380f1da5..c023ace4e 100644
--- a/drivers/net/virtio/virtio_rxtx_packed_avx.c
+++ b/drivers/net/virtio/virtio_rxtx_packed_avx.c
@@ -23,6 +23,24 @@
#define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
FLAGS_BITS_OFFSET)
+/* reference count offset in mbuf rearm data */
+#define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+/* segment number offset in mbuf rearm data */
+#define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
+ offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
+
+/* default rearm data */
+#define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
+ 1ULL << REFCNT_BITS_OFFSET)
+
+/* id bits offset in packed ring desc higher 64bits */
+#define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
+ offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
+
+/* net hdr short size mask */
+#define NET_HDR_MASK 0x3F
+
#define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
sizeof(struct vring_packed_desc))
#define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
@@ -47,6 +65,47 @@
for (iter = val; iter < num; iter++)
#endif
+static inline void
+virtio_xmit_cleanup_packed_vec(struct virtqueue *vq)
+{
+ struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
+ struct vq_desc_extra *dxp;
+ uint16_t used_idx, id, curr_id, free_cnt = 0;
+ uint16_t size = vq->vq_nentries;
+ struct rte_mbuf *mbufs[size];
+ uint16_t nb_mbuf = 0, i;
+
+ used_idx = vq->vq_used_cons_idx;
+
+ if (!desc_is_used(&desc[used_idx], vq))
+ return;
+
+ id = desc[used_idx].id;
+
+ do {
+ curr_id = used_idx;
+ dxp = &vq->vq_descx[used_idx];
+ used_idx += dxp->ndescs;
+ free_cnt += dxp->ndescs;
+
+ if (dxp->cookie != NULL) {
+ mbufs[nb_mbuf] = dxp->cookie;
+ dxp->cookie = NULL;
+ nb_mbuf++;
+ }
+
+ if (used_idx >= size) {
+ used_idx -= size;
+ vq->vq_packed.used_wrap_counter ^= 1;
+ }
+ } while (curr_id != id);
+
+ for (i = 0; i < nb_mbuf; i++)
+ rte_pktmbuf_free(mbufs[i]);
+
+ vq->vq_used_cons_idx = used_idx;
+ vq->vq_free_cnt += free_cnt;
+}
static inline void
virtio_update_batch_stats(struct virtnet_stats *stats,
@@ -60,6 +119,238 @@ virtio_update_batch_stats(struct virtnet_stats *stats,
stats->bytes += pkt_len3;
stats->bytes += pkt_len4;
}
+
+static inline int
+virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
+ struct rte_mbuf **tx_pkts)
+{
+ struct virtqueue *vq = txvq->vq;
+ uint16_t head_size = vq->hw->vtnet_hdr_size;
+ uint16_t idx = vq->vq_avail_idx;
+ struct virtio_net_hdr *hdr;
+ uint16_t i, cmp;
+
+ if (vq->vq_avail_idx & PACKED_BATCH_MASK)
+ return -1;
+
+ if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
+ return -1;
+
+ /* Load four mbufs rearm data */
+ RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
+ RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
+ __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
+
[dpdk-dev] [PATCH v8 6/9] net/virtio: reuse packed ring xmit functions
Move xmit offload and packed ring xmit enqueue function to header file.
These functions will be reused by packed ring vectorized Tx function.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 7b65d0b0a..cf18fe564 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -264,10 +264,6 @@ virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
return i;
}
-#ifndef DEFAULT_TX_FREE_THRESH
-#define DEFAULT_TX_FREE_THRESH 32
-#endif
-
static void
virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
{
@@ -562,68 +558,7 @@ virtio_tso_fix_cksum(struct rte_mbuf *m)
}
-/* avoid write operation when necessary, to lessen cache issues */
-#define ASSIGN_UNLESS_EQUAL(var, val) do { \
- if ((var) != (val)) \
- (var) = (val); \
-} while (0)
-
-#define virtqueue_clear_net_hdr(_hdr) do { \
- ASSIGN_UNLESS_EQUAL((_hdr)->csum_start, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->csum_offset, 0);\
- ASSIGN_UNLESS_EQUAL((_hdr)->flags, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->gso_type, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->gso_size, 0); \
- ASSIGN_UNLESS_EQUAL((_hdr)->hdr_len, 0);\
-} while (0)
-
-static inline void
-virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
- struct rte_mbuf *cookie,
- bool offload)
-{
- if (offload) {
- if (cookie->ol_flags & PKT_TX_TCP_SEG)
- cookie->ol_flags |= PKT_TX_TCP_CKSUM;
-
- switch (cookie->ol_flags & PKT_TX_L4_MASK) {
- case PKT_TX_UDP_CKSUM:
- hdr->csum_start = cookie->l2_len + cookie->l3_len;
- hdr->csum_offset = offsetof(struct rte_udp_hdr,
- dgram_cksum);
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- break;
-
- case PKT_TX_TCP_CKSUM:
- hdr->csum_start = cookie->l2_len + cookie->l3_len;
- hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum);
- hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
- break;
-
- default:
- ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
- ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
- ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
- break;
- }
- /* TCP Segmentation Offload */
- if (cookie->ol_flags & PKT_TX_TCP_SEG) {
- hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
- VIRTIO_NET_HDR_GSO_TCPV6 :
- VIRTIO_NET_HDR_GSO_TCPV4;
- hdr->gso_size = cookie->tso_segsz;
- hdr->hdr_len =
- cookie->l2_len +
- cookie->l3_len +
- cookie->l4_len;
- } else {
- ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
- ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
- ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
- }
- }
-}
static inline void
virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
@@ -725,102 +660,6 @@ virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx
*txvq,
virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
}
-static inline void
-virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
- uint16_t needed, int can_push, int in_order)
-{
- struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
- struct vq_desc_extra *dxp;
- struct virtqueue *vq = txvq->vq;
- struct vring_packed_desc *start_dp, *head_dp;
- uint16_t idx, id, head_idx, head_flags;
- int16_t head_size = vq->hw->vtnet_hdr_size;
- struct virtio_net_hdr *hdr;
- uint16_t prev;
- bool prepend_header = false;
-
- id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
-
- dxp = &vq->vq_descx[id];
- dxp->ndescs = needed;
- dxp->cookie = cookie;
-
- head_idx = vq->vq_avail_idx;
- idx = head_idx;
- prev = head_idx;
- start_dp = vq->vq_packed.ring.desc;
-
- head_dp = &vq->vq_packed.ring.desc[idx];
- head_flags = cookie->next ? VRING_DESC_F_NEXT : 0;
- head_flags |= vq->vq_packed.cached_flags;
-
- if (can_push) {
- /* prepend cannot fail, checked by caller */
- hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr
[dpdk-dev] [PATCH v8 3/9] net/virtio: inorder should depend on feature bit
Ring initialzation is different when inorder feature negotiated. This
action should dependent on negotiated feature bits.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 94ba7a3ec..e450477e8 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -989,6 +989,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
struct rte_mbuf *m;
uint16_t desc_idx;
int error, nbufs, i;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
PMD_INIT_FUNC_TRACE();
@@ -1018,7 +1019,7 @@ virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev,
uint16_t queue_idx)
virtio_rxq_rearm_vec(rxvq);
nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
}
- } else if (hw->use_inorder_rx) {
+ } else if (!vtpci_packed_queue(vq->hw) && in_order) {
if ((!virtqueue_full(vq))) {
uint16_t free_cnt = vq->vq_free_cnt;
struct rte_mbuf *pkts[free_cnt];
@@ -1133,7 +1134,7 @@ virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
PMD_INIT_FUNC_TRACE();
if (!vtpci_packed_queue(hw)) {
- if (hw->use_inorder_tx)
+ if (vtpci_with_feature(hw, VIRTIO_F_IN_ORDER))
vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
}
@@ -2046,7 +2047,7 @@ virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf
**tx_pkts,
struct virtio_hw *hw = vq->hw;
uint16_t hdr_size = hw->vtnet_hdr_size;
uint16_t nb_tx = 0;
- bool in_order = hw->use_inorder_tx;
+ bool in_order = vtpci_with_feature(hw, VIRTIO_F_IN_ORDER);
if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
return nb_tx;
--
2.17.1
[dpdk-dev] [PATCH v8 5/9] net/virtio: add vectorized packed ring Rx path
Optimize packed ring Rx path when AVX512 enabled and mergeable
buffer/Rx LRO offloading are not required. Solution of optimization
is pretty like vhost, is that split path into batch and single
functions. Batch function is further optimized by vector instructions.
Also pad desc extra structure to 16 bytes aligned, thus four elements
will be saved in one batch.
Signed-off-by: Marvin Liu
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index 4b69827ab..de0b00e50 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -36,6 +36,41 @@ SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) +=
virtio_rxtx_simple_altivec.c
else ifneq ($(filter y,$(CONFIG_RTE_ARCH_ARM) $(CONFIG_RTE_ARCH_ARM64)),)
SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_simple_neon.c
endif
+
+ifneq ($(FORCE_DISABLE_AVX512), y)
+ CC_AVX512_SUPPORT=\
+ $(shell $(CC) -march=native -dM -E - &1 | \
+ sed '/./{H;$$!d} ; x ; /AVX512F/!d; /AVX512BW/!d; /AVX512VL/!d' | \
+ grep -q AVX512 && echo 1)
+endif
+
+ifeq ($(CC_AVX512_SUPPORT), 1)
+CFLAGS += -DCC_AVX512_SUPPORT
+SRCS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio_rxtx_packed_avx.c
+
+ifeq ($(RTE_TOOLCHAIN), gcc)
+ifeq ($(shell test $(GCC_VERSION) -ge 83 && echo 1), 1)
+CFLAGS += -DVIRTIO_GCC_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), clang)
+ifeq ($(shell test $(CLANG_MAJOR_VERSION)$(CLANG_MINOR_VERSION) -ge 37 && echo
1), 1)
+CFLAGS += -DVIRTIO_CLANG_UNROLL_PRAGMA
+endif
+endif
+
+ifeq ($(RTE_TOOLCHAIN), icc)
+ifeq ($(shell test $(ICC_MAJOR_VERSION) -ge 16 && echo 1), 1)
+CFLAGS += -DVIRTIO_ICC_UNROLL_PRAGMA
+endif
+endif
+
+CFLAGS_virtio_rxtx_packed_avx.o += -mavx512f -mavx512bw -mavx512vl
+ifeq ($(shell test $(GCC_VERSION) -ge 100 && echo 1), 1)
+CFLAGS_virtio_rxtx_packed_avx.o += -Wno-zero-length-bounds
+endif
+endif
endif
ifeq ($(CONFIG_RTE_VIRTIO_USER),y)
diff --git a/drivers/net/virtio/meson.build b/drivers/net/virtio/meson.build
index ce3525ef5..39b3605d9 100644
--- a/drivers/net/virtio/meson.build
+++ b/drivers/net/virtio/meson.build
@@ -10,6 +10,20 @@ deps += ['kvargs', 'bus_pci']
dpdk_conf.set('RTE_LIBRTE_VIRTIO_INC_VECTOR', 1)
if arch_subdir == 'x86'
+ if '-mno-avx512f' not in machine_args
+ if cc.has_argument('-mavx512f') and
cc.has_argument('-mavx512vl') and cc.has_argument('-mavx512bw')
+ cflags += ['-mavx512f', '-mavx512bw', '-mavx512vl']
+ cflags += ['-DCC_AVX512_SUPPORT']
+ if (toolchain == 'gcc' and
cc.version().version_compare('>=8.3.0'))
+ cflags += '-DVHOST_GCC_UNROLL_PRAGMA'
+ elif (toolchain == 'clang' and
cc.version().version_compare('>=3.7.0'))
+ cflags += '-DVHOST_CLANG_UNROLL_PRAGMA'
+ elif (toolchain == 'icc' and
cc.version().version_compare('>=16.0.0'))
+ cflags += '-DVHOST_ICC_UNROLL_PRAGMA'
+ endif
+ sources += files('virtio_rxtx_packed_avx.c')
+ endif
+ endif
sources += files('virtio_rxtx_simple_sse.c')
elif arch_subdir == 'ppc'
sources += files('virtio_rxtx_simple_altivec.c')
diff --git a/drivers/net/virtio/virtio_ethdev.h
b/drivers/net/virtio/virtio_ethdev.h
index febaf17a8..5c112cac7 100644
--- a/drivers/net/virtio/virtio_ethdev.h
+++ b/drivers/net/virtio/virtio_ethdev.h
@@ -105,6 +105,9 @@ uint16_t virtio_xmit_pkts_inorder(void *tx_queue, struct
rte_mbuf **tx_pkts,
uint16_t virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts);
+uint16_t virtio_recv_pkts_packed_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+
int eth_virtio_dev_init(struct rte_eth_dev *eth_dev);
void virtio_interrupt_handler(void *param);
diff --git a/drivers/net/virtio/virtio_rxtx.c b/drivers/net/virtio/virtio_rxtx.c
index 84f4cf946..7b65d0b0a 100644
--- a/drivers/net/virtio/virtio_rxtx.c
+++ b/drivers/net/virtio/virtio_rxtx.c
@@ -1246,7 +1246,6 @@ virtio_rx_offload(struct rte_mbuf *m, struct
virtio_net_hdr *hdr)
return 0;
}
-#define VIRTIO_MBUF_BURST_SZ 64
#define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
uint16_t
virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
@@ -2329,3 +2328,11 @@ virtio_xmit_pkts_inorder(void *tx_queue,
return nb_tx;
}
+
+__rte_weak uint16_t
+virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
+ struct rte_mbuf **rx_pkts __rte_unused,
+ uint16_t nb_pkts
