[dpdk-dev] [PATCH RFC v2 10/12] lib/librte_vhost: vhost-user memory region map
(2014/12/11 6:37), Huawei Xie wrote:
> deals with vhost user memory map/unmap alignment
>
> Signed-off-by: Huawei Xie
> ---
> lib/librte_vhost/rte_virtio_net.h | 2 +
> lib/librte_vhost/vhost-net.h | 2 -
> lib/librte_vhost/vhost_user/vhost-net-user.h | 3 +-
> lib/librte_vhost/vhost_user/virtio-net-user.c | 105
> --
> 4 files changed, 100 insertions(+), 12 deletions(-)
>
> diff --git a/lib/librte_vhost/rte_virtio_net.h
> b/lib/librte_vhost/rte_virtio_net.h
> index 00b1328..77db80b 100644
> --- a/lib/librte_vhost/rte_virtio_net.h
> +++ b/lib/librte_vhost/rte_virtio_net.h
> @@ -48,6 +48,8 @@
> #include
> #include
>
> +#define VHOST_MEMORY_MAX_NREGIONS 8
> +
> /* Used to indicate that the device is running on a data core */
> #define VIRTIO_DEV_RUNNING 1
>
> diff --git a/lib/librte_vhost/vhost-net.h b/lib/librte_vhost/vhost-net.h
> index f9ec40b..ec2584f 100644
> --- a/lib/librte_vhost/vhost-net.h
> +++ b/lib/librte_vhost/vhost-net.h
> @@ -43,8 +43,6 @@
>
> #include "rte_virtio_net.h"
>
> -#define VHOST_MEMORY_MAX_NREGIONS 8
> -
> extern struct vhost_net_device_ops const *ops;
>
> /* Macros for printing using RTE_LOG */
> diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.h
> b/lib/librte_vhost/vhost_user/vhost-net-user.h
> index c138844..f4c9d01 100644
> --- a/lib/librte_vhost/vhost_user/vhost-net-user.h
> +++ b/lib/librte_vhost/vhost_user/vhost-net-user.h
> @@ -37,6 +37,7 @@
> #include
> #include
>
> +#include "rte_virtio_net.h"
> #include "fd_man.h"
>
> struct vhost_server {
> @@ -47,8 +48,6 @@ struct vhost_server {
>
> /* refer to hw/virtio/vhost-user.c */
>
> -#define VHOST_MEMORY_MAX_NREGIONS8
> -
> typedef enum VhostUserRequest {
> VHOST_USER_NONE = 0,
> VHOST_USER_GET_FEATURES = 1,
> diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c
> b/lib/librte_vhost/vhost_user/virtio-net-user.c
> index ad59fcc..3aecb17 100644
> --- a/lib/librte_vhost/vhost_user/virtio-net-user.c
> +++ b/lib/librte_vhost/vhost_user/virtio-net-user.c
> @@ -36,7 +36,11 @@
> #include
> #include
> #include
> +#include
> +#include
> +#include
>
> +#include
> #include
>
> #include "virtio-net.h"
> @@ -44,13 +48,56 @@
> #include "vhost-net-user.h"
> #include "vhost-net.h"
>
> +struct orig_region_map {
> + int fd;
> + uint64_t mapped_address;
> + uint64_t mapped_size;
> + uint64_t blksz;
> +};
> +
> +#define orig_region(ptr, nregions) (struct orig_region_map
> *)RTE_PTR_ADD(ptr, sizeof(struct virtio_memory) + sizeof(struct
> virtio_memory_regions) * (nregions))
> +
> +static uint64_t
> +get_blk_size(int fd)
> +{
> + struct stat stat;
> + fstat(fd, );
> + return (uint64_t)stat.st_blksize;
> +}
I've also confirmed we can get hugepage size of the fd using st_blksize.
If someone wants to run QEMU on 2MB hugepage, but DPDK backend is on
1GB, even in such a case, we will also be able to mmap and munmap QEMU
backend memory correctly.
So I guess using st_blksize is smart workaround not to hit munmap issue.
> +
> int
> user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg)
> {
> - unsigned int idx;
> struct VhostUserMemory memory = pmsg->payload.memory;
> struct virtio_memory_regions regions[VHOST_MEMORY_MAX_NREGIONS];
> - uint64_t mapped_address, base_address = 0;
> + uint64_t mapped_address, mapped_size, base_address = 0;
> + struct virtio_net *dev;
> + unsigned int idx = 0;
> + struct orig_region_map tmp[VHOST_MEMORY_MAX_NREGIONS] =
> + { [0 ... VHOST_MEMORY_MAX_NREGIONS - 1] = { 0 } };
> + struct orig_region_map *region;
> + uint64_t alignment;
> + int ret;
> +
> + /* unmap old memory regions one by one*/
> + dev = get_device(ctx);
> + if (dev->mem) {
> + region = orig_region(dev->mem, dev->mem->nregions);
> + for (idx = 0; idx < dev->mem->nregions; idx++) {
> + if (region[idx].mapped_address) {
> + alignment = region[idx].blksz;
> + printf("Freeing %p\n",
> + (void
> *)(uintptr_t)region[idx].mapped_address);
> + ret = munmap((void
> *)RTE_ALIGN_FLOOR(region[idx].mapped_address, alignment),
> + RTE_ALIGN_CEIL(region[idx].mapped_size,
> alignment));
> + printf("munmap ret= %d\n", ret);
> + printf("close file %d\n", region[idx].fd);
> + close(region[idx].fd);
> + }
> + }
> + free(dev->mem);
> + dev->mem = NULL;
> + }
>
> for (idx = 0; idx < memory.nregions; idx++) {
> if (memory.regions[idx].guest_phys_addr == 0)
> @@ -73,22 +120,30 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct
> VhostUserMsg
[dpdk-dev] [PATCH RFC v2 10/12] lib/librte_vhost: vhost-user memory region map
deals with vhost user memory map/unmap alignment
Signed-off-by: Huawei Xie
---
lib/librte_vhost/rte_virtio_net.h | 2 +
lib/librte_vhost/vhost-net.h | 2 -
lib/librte_vhost/vhost_user/vhost-net-user.h | 3 +-
lib/librte_vhost/vhost_user/virtio-net-user.c | 105 --
4 files changed, 100 insertions(+), 12 deletions(-)
diff --git a/lib/librte_vhost/rte_virtio_net.h
b/lib/librte_vhost/rte_virtio_net.h
index 00b1328..77db80b 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -48,6 +48,8 @@
#include
#include
+#define VHOST_MEMORY_MAX_NREGIONS 8
+
/* Used to indicate that the device is running on a data core */
#define VIRTIO_DEV_RUNNING 1
diff --git a/lib/librte_vhost/vhost-net.h b/lib/librte_vhost/vhost-net.h
index f9ec40b..ec2584f 100644
--- a/lib/librte_vhost/vhost-net.h
+++ b/lib/librte_vhost/vhost-net.h
@@ -43,8 +43,6 @@
#include "rte_virtio_net.h"
-#define VHOST_MEMORY_MAX_NREGIONS 8
-
extern struct vhost_net_device_ops const *ops;
/* Macros for printing using RTE_LOG */
diff --git a/lib/librte_vhost/vhost_user/vhost-net-user.h
b/lib/librte_vhost/vhost_user/vhost-net-user.h
index c138844..f4c9d01 100644
--- a/lib/librte_vhost/vhost_user/vhost-net-user.h
+++ b/lib/librte_vhost/vhost_user/vhost-net-user.h
@@ -37,6 +37,7 @@
#include
#include
+#include "rte_virtio_net.h"
#include "fd_man.h"
struct vhost_server {
@@ -47,8 +48,6 @@ struct vhost_server {
/* refer to hw/virtio/vhost-user.c */
-#define VHOST_MEMORY_MAX_NREGIONS8
-
typedef enum VhostUserRequest {
VHOST_USER_NONE = 0,
VHOST_USER_GET_FEATURES = 1,
diff --git a/lib/librte_vhost/vhost_user/virtio-net-user.c
b/lib/librte_vhost/vhost_user/virtio-net-user.c
index ad59fcc..3aecb17 100644
--- a/lib/librte_vhost/vhost_user/virtio-net-user.c
+++ b/lib/librte_vhost/vhost_user/virtio-net-user.c
@@ -36,7 +36,11 @@
#include
#include
#include
+#include
+#include
+#include
+#include
#include
#include "virtio-net.h"
@@ -44,13 +48,56 @@
#include "vhost-net-user.h"
#include "vhost-net.h"
+struct orig_region_map {
+ int fd;
+ uint64_t mapped_address;
+ uint64_t mapped_size;
+ uint64_t blksz;
+};
+
+#define orig_region(ptr, nregions) (struct orig_region_map *)RTE_PTR_ADD(ptr,
sizeof(struct virtio_memory) + sizeof(struct virtio_memory_regions) *
(nregions))
+
+static uint64_t
+get_blk_size(int fd)
+{
+ struct stat stat;
+ fstat(fd, );
+ return (uint64_t)stat.st_blksize;
+}
+
int
user_set_mem_table(struct vhost_device_ctx ctx, struct VhostUserMsg *pmsg)
{
- unsigned int idx;
struct VhostUserMemory memory = pmsg->payload.memory;
struct virtio_memory_regions regions[VHOST_MEMORY_MAX_NREGIONS];
- uint64_t mapped_address, base_address = 0;
+ uint64_t mapped_address, mapped_size, base_address = 0;
+ struct virtio_net *dev;
+ unsigned int idx = 0;
+ struct orig_region_map tmp[VHOST_MEMORY_MAX_NREGIONS] =
+ { [0 ... VHOST_MEMORY_MAX_NREGIONS - 1] = { 0 } };
+ struct orig_region_map *region;
+ uint64_t alignment;
+ int ret;
+
+ /* unmap old memory regions one by one*/
+ dev = get_device(ctx);
+ if (dev->mem) {
+ region = orig_region(dev->mem, dev->mem->nregions);
+ for (idx = 0; idx < dev->mem->nregions; idx++) {
+ if (region[idx].mapped_address) {
+ alignment = region[idx].blksz;
+ printf("Freeing %p\n",
+ (void
*)(uintptr_t)region[idx].mapped_address);
+ ret = munmap((void
*)RTE_ALIGN_FLOOR(region[idx].mapped_address, alignment),
+ RTE_ALIGN_CEIL(region[idx].mapped_size,
alignment));
+ printf("munmap ret= %d\n", ret);
+ printf("close file %d\n", region[idx].fd);
+ close(region[idx].fd);
+ }
+ }
+ free(dev->mem);
+ dev->mem = NULL;
+ }
for (idx = 0; idx < memory.nregions; idx++) {
if (memory.regions[idx].guest_phys_addr == 0)
@@ -73,22 +120,30 @@ user_set_mem_table(struct vhost_device_ctx ctx, struct
VhostUserMsg *pmsg)
memory.regions[idx].userspace_addr;
/* This is ugly */
+ mapped_size = regions[idx].memory_size +
+ memory.regions[idx].mmap_offset;
mapped_address = (uint64_t)(uintptr_t)mmap(NULL,
- regions[idx].memory_size +
- memory.regions[idx].mmap_offset,
+ mapped_size,
PROT_READ | PROT_WRITE, MAP_SHARED,
pmsg->fds[idx],
0);
+
