The AF_UNIX transport can accept multiple client connections on a server
socket. Each connection instantiates a separate vhost-user device,
which is stored as a vhost_user_connection. This behavior is specific
to AF_UNIX and other transports may not support N connections per
socket endpoint.
Move struct vhost_user_connection to trans_af_unix.c and
conn_list/conn_mutex into struct af_unix_socket.
Signed-off-by: Nikos Dragazis <ndraga...@arrikto.com>
Signed-off-by: Stefan Hajnoczi <stefa...@redhat.com>
---
lib/librte_vhost/socket.c | 54 +++---------------------------
lib/librte_vhost/trans_af_unix.c | 72 ++++++++++++++++++++++++++++++++++++----
lib/librte_vhost/vhost.h | 19 ++---------
3 files changed, 74 insertions(+), 71 deletions(-)
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index df6d707..976343c 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -341,13 +341,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
vhost_user_socket_mem_free(vsocket);
goto out;
}
- TAILQ_INIT(&vsocket->conn_list);
- ret = pthread_mutex_init(&vsocket->conn_mutex, NULL);
- if (ret) {
- RTE_LOG(ERR, VHOST_CONFIG,
- "error: failed to init connection mutex\n");
- goto out_free;
- }
vsocket->dequeue_zero_copy = flags & RTE_VHOST_USER_DEQUEUE_ZERO_COPY;
/*
@@ -395,7 +388,7 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
RTE_LOG(ERR, VHOST_CONFIG,
"Postcopy requested but not compiled\n");
ret = -1;
- goto out_mutex;
+ goto out_free;
#endif
}
@@ -403,14 +396,14 @@ rte_vhost_driver_register(const char *path, uint64_t
flags)
vsocket->reconnect = !(flags & RTE_VHOST_USER_NO_RECONNECT);
if (vsocket->reconnect && reconn_tid == 0) {
if (vhost_user_reconnect_init() != 0)
- goto out_mutex;
+ goto out_free;
}
} else {
vsocket->is_server = true;
}
ret = trans_ops->socket_init(vsocket, flags);
if (ret < 0) {
- goto out_mutex;
+ goto out_free;
}
vhost_user.vsockets[vhost_user.vsocket_cnt++] = vsocket;
@@ -418,11 +411,6 @@ rte_vhost_driver_register(const char *path, uint64_t flags)
pthread_mutex_unlock(&vhost_user.mutex);
return ret;
-out_mutex:
- if (pthread_mutex_destroy(&vsocket->conn_mutex)) {
- RTE_LOG(ERR, VHOST_CONFIG,
- "error: failed to destroy connection mutex\n");
- }
out_free:
vhost_user_socket_mem_free(vsocket);
out:
@@ -439,51 +427,19 @@ rte_vhost_driver_unregister(const char *path)
{
int i;
int count;
- struct vhost_user_connection *conn, *next;
if (path == NULL)
return -1;
-again:
pthread_mutex_lock(&vhost_user.mutex);
for (i = 0; i < vhost_user.vsocket_cnt; i++) {
struct vhost_user_socket *vsocket = vhost_user.vsockets[i];
if (!strcmp(vsocket->path, path)) {
- pthread_mutex_lock(&vsocket->conn_mutex);
- for (conn = TAILQ_FIRST(&vsocket->conn_list);
- conn != NULL;
- conn = next) {
- next = TAILQ_NEXT(conn, next);
-
- /*
- * If r/wcb is executing, release the
- * conn_mutex lock, and try again since
- * the r/wcb may use the conn_mutex lock.
- */
- if (fdset_try_del(&vhost_user.fdset,
- conn->connfd) == -1) {
- pthread_mutex_unlock(
- &vsocket->conn_mutex);
- pthread_mutex_unlock(&vhost_user.mutex);
- goto again;
- }
-
- RTE_LOG(INFO, VHOST_CONFIG,
- "free connfd = %d for device '%s'\n",
- conn->connfd, path);
- close(conn->connfd);
- vhost_destroy_device(conn->vid);
- TAILQ_REMOVE(&vsocket->conn_list, conn, next);
- free(conn);
- }
- pthread_mutex_unlock(&vsocket->conn_mutex);
-
vsocket->trans_ops->socket_cleanup(vsocket);
-
- pthread_mutex_destroy(&vsocket->conn_mutex);
- vhost_user_socket_mem_free(vsocket);
+ free(vsocket->path);
+ free(vsocket);
count = --vhost_user.vsocket_cnt;
vhost_user.vsockets[i] = vhost_user.vsockets[count];
diff --git a/lib/librte_vhost/trans_af_unix.c b/lib/librte_vhost/trans_af_unix.c
index 93d11f7..58fc9e2 100644
--- a/lib/librte_vhost/trans_af_unix.c
+++ b/lib/librte_vhost/trans_af_unix.c
@@ -15,8 +15,20 @@
#define MAX_VIRTIO_BACKLOG 128
+TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
+
+struct vhost_user_connection {
+ struct vhost_user_socket *vsocket;
+ int connfd;
+ int vid;
+
+ TAILQ_ENTRY(vhost_user_connection) next;
+};
+
struct af_unix_socket {
struct vhost_user_socket socket; /* must be the first field! */
+ struct vhost_user_connection_list conn_list;
+ pthread_mutex_t conn_mutex;
int socket_fd;
struct sockaddr_un un;
};
@@ -131,6 +143,8 @@ send_fd_message(int sockfd, char *buf, int buflen, int
*fds, int fd_num)
static void
vhost_user_add_connection(int fd, struct vhost_user_socket *vsocket)
{
+ struct af_unix_socket *af_vsocket =
+ container_of(vsocket, struct af_unix_socket, socket);
int vid;
size_t size;
struct vhost_user_connection *conn;
@@ -188,9 +202,9 @@ vhost_user_add_connection(int fd, struct vhost_user_socket
*vsocket)
goto err_cleanup;
}
- pthread_mutex_lock(&vsocket->conn_mutex);
- TAILQ_INSERT_TAIL(&vsocket->conn_list, conn, next);
- pthread_mutex_unlock(&vsocket->conn_mutex);
+ pthread_mutex_lock(&af_vsocket->conn_mutex);
+ TAILQ_INSERT_TAIL(&af_vsocket->conn_list, conn, next);
+ pthread_mutex_unlock(&af_vsocket->conn_mutex);
fdset_pipe_notify(&vhost_user.fdset);
return;
@@ -221,6 +235,8 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
{
struct vhost_user_connection *conn = dat;
struct vhost_user_socket *vsocket = conn->vsocket;
+ struct af_unix_socket *af_vsocket =
+ container_of(vsocket, struct af_unix_socket, socket);
int ret;
ret = vhost_user_msg_handler(conn->vid, connfd);
@@ -238,9 +254,9 @@ vhost_user_read_cb(int connfd, void *dat, int *remove)
vhost_destroy_device(conn->vid);
- pthread_mutex_lock(&vsocket->conn_mutex);
- TAILQ_REMOVE(&vsocket->conn_list, conn, next);
- pthread_mutex_unlock(&vsocket->conn_mutex);
+ pthread_mutex_lock(&af_vsocket->conn_mutex);
+ TAILQ_REMOVE(&af_vsocket->conn_list, conn, next);
+ pthread_mutex_unlock(&af_vsocket->conn_mutex);
free(conn);
@@ -512,6 +528,18 @@ static int
af_unix_socket_init(struct vhost_user_socket *vsocket,
uint64_t flags __rte_unused)
{
+ struct af_unix_socket *af_vsocket =
+ container_of(vsocket, struct af_unix_socket, socket);
+ int ret;
+
+ TAILQ_INIT(&af_vsocket->conn_list);
+ ret = pthread_mutex_init(&af_vsocket->conn_mutex, NULL);
+ if (ret) {
+ RTE_LOG(ERR, VHOST_CONFIG,
+ "error: failed to init connection mutex\n");
+ return -1;
+ }
+
return create_unix_socket(vsocket);
}
@@ -520,6 +548,7 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
{
struct af_unix_socket *af_vsocket =
container_of(vsocket, struct af_unix_socket, socket);
+ struct vhost_user_connection *conn, *next;
if (vsocket->is_server) {
fdset_del(&vhost_user.fdset, af_vsocket->socket_fd);
@@ -528,6 +557,37 @@ af_unix_socket_cleanup(struct vhost_user_socket *vsocket)
} else if (vsocket->reconnect) {
vhost_user_remove_reconnect(vsocket);
}
+
+again:
+ pthread_mutex_lock(&af_vsocket->conn_mutex);
+ for (conn = TAILQ_FIRST(&af_vsocket->conn_list);
+ conn != NULL;
+ conn = next) {
+ next = TAILQ_NEXT(conn, next);
+
+ /*
+ * If r/wcb is executing, release the
+ * conn_mutex lock, and try again since
+ * the r/wcb may use the conn_mutex lock.
+ */
+ if (fdset_try_del(&vhost_user.fdset,
+ conn->connfd) == -1) {
+ pthread_mutex_unlock(
+ &af_vsocket->conn_mutex);
+ goto again;
+ }
+
+ RTE_LOG(INFO, VHOST_CONFIG,
+ "free connfd = %d for device '%s'\n",
+ conn->connfd, vsocket->path);
+ close(conn->connfd);
+ vhost_destroy_device(conn->vid);
+ TAILQ_REMOVE(&af_vsocket->conn_list, conn, next);
+ free(conn);
+ }
+ pthread_mutex_unlock(&af_vsocket->conn_mutex);
+
+ pthread_mutex_destroy(&af_vsocket->conn_mutex);
}
static int
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index c74753b..5c3987d 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -404,13 +404,10 @@ struct virtio_net {
struct rte_vhost_user_extern_ops extern_ops;
} __rte_cache_aligned;
-/* The vhost_user, vhost_user_socket, vhost_user_connection, and reconnect
- * declarations are temporary measures for moving AF_UNIX code into
- * trans_af_unix.c. They will be cleaned up as socket.c is untangled from
- * trans_af_unix.c.
+/* The vhost_user, vhost_user_socket, and reconnect declarations are temporary
+ * measures for moving AF_UNIX code into trans_af_unix.c. They will be cleaned
+ * up as socket.c is untangled from trans_af_unix.c.
*/
-TAILQ_HEAD(vhost_user_connection_list, vhost_user_connection);
-
/*
* Every time rte_vhost_driver_register() is invoked, an associated
* vhost_user_socket struct will be created.
@@ -421,8 +418,6 @@ TAILQ_HEAD(vhost_user_connection_list,
vhost_user_connection);
* struct.
*/
struct vhost_user_socket {
- struct vhost_user_connection_list conn_list;
- pthread_mutex_t conn_mutex;
char *path;
bool is_server;
bool reconnect;
@@ -453,14 +448,6 @@ struct vhost_user_socket {
struct vhost_transport_ops const *trans_ops;
};
-struct vhost_user_connection {
- struct vhost_user_socket *vsocket;
- int connfd;
- int vid;
-
- TAILQ_ENTRY(vhost_user_connection) next;
-};
-
#define MAX_VHOST_SOCKET 1024
struct vhost_user {
struct vhost_user_socket *vsockets[MAX_VHOST_SOCKET];
--
2.7.4
