On Mon, Jun 14, 2010 at 9:53 AM, Anthony Liguori <anth...@codemonkey.ws> wrote: > On 06/04/2010 04:45 PM, Cam Macdonell wrote: >> >> this code is a standalone server which will pass file descriptors for the >> shared >> memory region and eventfds to support interrupts between guests using >> inter-VM >> shared memory. >> --- >> contrib/ivshmem-server/Makefile | 16 ++ >> contrib/ivshmem-server/README | 30 +++ >> contrib/ivshmem-server/ivshmem_server.c | 353 >> +++++++++++++++++++++++++++++++ >> contrib/ivshmem-server/send_scm.c | 208 ++++++++++++++++++ >> contrib/ivshmem-server/send_scm.h | 19 ++ >> 5 files changed, 626 insertions(+), 0 deletions(-) >> create mode 100644 contrib/ivshmem-server/Makefile >> create mode 100644 contrib/ivshmem-server/README >> create mode 100644 contrib/ivshmem-server/ivshmem_server.c >> create mode 100644 contrib/ivshmem-server/send_scm.c >> create mode 100644 contrib/ivshmem-server/send_scm.h >> >> diff --git a/contrib/ivshmem-server/Makefile >> b/contrib/ivshmem-server/Makefile >> new file mode 100644 >> index 0000000..da40ffa >> --- /dev/null >> +++ b/contrib/ivshmem-server/Makefile >> @@ -0,0 +1,16 @@ >> +CC = gcc >> +CFLAGS = -O3 -Wall -Werror >> +LIBS = -lrt >> + >> +# a very simple makefile to build the inter-VM shared memory server >> + >> +all: ivshmem_server >> + >> +.c.o: >> + $(CC) $(CFLAGS) -c $^ -o $@ >> + >> +ivshmem_server: ivshmem_server.o send_scm.o >> + $(CC) $(CFLAGS) -o $@ $^ $(LIBS) >> + >> +clean: >> + rm -f *.o ivshmem_server >> diff --git a/contrib/ivshmem-server/README b/contrib/ivshmem-server/README >> new file mode 100644 >> index 0000000..b1fc2a2 >> --- /dev/null >> +++ b/contrib/ivshmem-server/README >> @@ -0,0 +1,30 @@ >> +Using the ivshmem shared memory server >> +-------------------------------------- >> + >> +This server is only supported on Linux. >> + >> +To use the shared memory server, first compile it. Running 'make' should >> +accomplish this. An executable named 'ivshmem_server' will be built. >> + >> +to display the options run: >> + >> +./ivshmem_server -h >> + >> +Options >> +------- >> + >> + -h print help message >> + >> + -p<path on host> >> + unix socket to listen on. The qemu-kvm chardev needs to connect >> on >> + this socket. (default: '/tmp/ivshmem_socket') >> + >> + -s<string> >> + POSIX shared object to create that is the shared memory (default: >> 'ivshmem') >> + >> + -m<#> >> + size of the POSIX object in MBs (default: 1) >> + >> + -n<#> >> + number of eventfds for each guest. This number must match the >> + 'vectors' argument passed the ivshmem device. (default: 1) >> diff --git a/contrib/ivshmem-server/ivshmem_server.c >> b/contrib/ivshmem-server/ivshmem_server.c >> new file mode 100644 >> index 0000000..e0a7b98 >> --- /dev/null >> +++ b/contrib/ivshmem-server/ivshmem_server.c >> > > There's no licensing here. I don't think this belongs in the qemu tree > either to be honest. If it were to be included, it ought to use all of the > existing qemu infrastructure like the other qemu-* tools.
For the time being, I'm willing to leave it out and host it externally. > > Regards, > > Anthony Liguori > >> @@ -0,0 +1,353 @@ >> +/* >> + * A stand-alone shared memory server for inter-VM shared memory for KVM >> +*/ >> + >> +#include<errno.h> >> +#include<string.h> >> +#include<sys/types.h> >> +#include<sys/socket.h> >> +#include<sys/un.h> >> +#include<unistd.h> >> +#include<sys/types.h> >> +#include<sys/stat.h> >> +#include<fcntl.h> >> +#include<sys/eventfd.h> >> +#include<sys/mman.h> >> +#include<sys/select.h> >> +#include<stdio.h> >> +#include<stdlib.h> >> +#include "send_scm.h" >> + >> +#define DEFAULT_SOCK_PATH "/tmp/ivshmem_socket" >> +#define DEFAULT_SHM_OBJ "ivshmem" >> + >> +#define DEBUG 1 >> + >> +typedef struct server_state { >> + vmguest_t *live_vms; >> + int nr_allocated_vms; >> + int shm_size; >> + long live_count; >> + long total_count; >> + int shm_fd; >> + char * path; >> + char * shmobj; >> + int maxfd, conn_socket; >> + long msi_vectors; >> +} server_state_t; >> + >> +void usage(char const *prg); >> +int find_set(fd_set * readset, int max); >> +void print_vec(server_state_t * s, const char * c); >> + >> +void add_new_guest(server_state_t * s); >> +void parse_args(int argc, char **argv, server_state_t * s); >> +int create_listening_socket(char * path); >> + >> +int main(int argc, char ** argv) >> +{ >> + fd_set readset; >> + server_state_t * s; >> + >> + s = (server_state_t *)calloc(1, sizeof(server_state_t)); >> + >> + s->live_count = 0; >> + s->total_count = 0; >> + parse_args(argc, argv, s); >> + >> + /* open shared memory file */ >> + if ((s->shm_fd = shm_open(s->shmobj, O_CREAT|O_RDWR, S_IRWXU))< 0) >> + { >> + fprintf(stderr, "kvm_ivshmem: could not open shared file\n"); >> + exit(-1); >> + } >> + >> + ftruncate(s->shm_fd, s->shm_size); >> + >> + s->conn_socket = create_listening_socket(s->path); >> + >> + s->maxfd = s->conn_socket; >> + >> + for(;;) { >> + int ret, handle, i; >> + char buf[1024]; >> + >> + print_vec(s, "vm_sockets"); >> + >> + FD_ZERO(&readset); >> + /* conn socket is in Live_vms at posn 0 */ >> + FD_SET(s->conn_socket,&readset); >> + for (i = 0; i< s->total_count; i++) { >> + if (s->live_vms[i].alive != 0) { >> + FD_SET(s->live_vms[i].sockfd,&readset); >> + } >> + } >> + >> + printf("\nWaiting (maxfd = %d)\n", s->maxfd); >> + >> + ret = select(s->maxfd + 1,&readset, NULL, NULL, NULL); >> + >> + if (ret == -1) { >> + perror("select()"); >> + } >> + >> + handle = find_set(&readset, s->maxfd + 1); >> + if (handle == -1) continue; >> + >> + if (handle == s->conn_socket) { >> + >> + printf("[NC] new connection\n"); >> + FD_CLR(s->conn_socket,&readset); >> + >> + /* The Total_count is equal to the new guests VM ID */ >> + add_new_guest(s); >> + >> + /* update our the maximum file descriptor number */ >> + s->maxfd = s->live_vms[s->total_count - 1].sockfd> s->maxfd >> ? >> + s->live_vms[s->total_count - 1].sockfd : >> s->maxfd; >> + >> + s->live_count++; >> + printf("Live_count is %ld\n", s->live_count); >> + >> + } else { >> + /* then we have received a disconnection */ >> + int recv_ret; >> + long i, j; >> + long deadposn = -1; >> + >> + recv_ret = recv(handle, buf, 1, 0); >> + >> + printf("[DC] recv returned %d\n", recv_ret); >> + >> + /* find the dead VM in our list and move it do the dead list. >> */ >> + for (i = 0; i< s->total_count; i++) { >> + if (s->live_vms[i].sockfd == handle) { >> + deadposn = i; >> + s->live_vms[i].alive = 0; >> + close(s->live_vms[i].sockfd); >> + >> + for (j = 0; j< s->msi_vectors; j++) { >> + close(s->live_vms[i].efd[j]); >> + } >> + >> + free(s->live_vms[i].efd); >> + s->live_vms[i].sockfd = -1; >> + break; >> + } >> + } >> + >> + for (j = 0; j< s->total_count; j++) { >> + /* update remaining clients that one client has left/died >> */ >> + if (s->live_vms[j].alive) { >> + printf("[UD] sending kill of fd[%ld] to %ld\n", >> + deadposn, >> j); >> + sendKill(s->live_vms[j].sockfd, deadposn, >> sizeof(deadposn)); >> + } >> + } >> + >> + s->live_count--; >> + >> + /* close the socket for the departed VM */ >> + close(handle); >> + } >> + >> + } >> + >> + return 0; >> +} >> + >> +void add_new_guest(server_state_t * s) { >> + >> + struct sockaddr_un remote; >> + socklen_t t = sizeof(remote); >> + long i, j; >> + int vm_sock; >> + long new_posn; >> + long neg1 = -1; >> + >> + vm_sock = accept(s->conn_socket, (struct sockaddr *)&remote,&t); >> + >> + if ( vm_sock == -1 ) { >> + perror("accept"); >> + exit(1); >> + } >> + >> + new_posn = s->total_count; >> + >> + if (new_posn == s->nr_allocated_vms) { >> + printf("increasing vm slots\n"); >> + s->nr_allocated_vms = s->nr_allocated_vms * 2; >> + if (s->nr_allocated_vms< 16) >> + s->nr_allocated_vms = 16; >> + s->live_vms = realloc(s->live_vms, >> + s->nr_allocated_vms * sizeof(vmguest_t)); >> + >> + if (s->live_vms == NULL) { >> + fprintf(stderr, "realloc failed - quitting\n"); >> + exit(-1); >> + } >> + } >> + >> + s->live_vms[new_posn].posn = new_posn; >> + printf("[NC] Live_vms[%ld]\n", new_posn); >> + s->live_vms[new_posn].efd = (int *) malloc(sizeof(int)); >> + for (i = 0; i< s->msi_vectors; i++) { >> + s->live_vms[new_posn].efd[i] = eventfd(0, 0); >> + printf("\tefd[%ld] = %d\n", i, s->live_vms[new_posn].efd[i]); >> + } >> + s->live_vms[new_posn].sockfd = vm_sock; >> + s->live_vms[new_posn].alive = 1; >> + >> + >> + sendPosition(vm_sock, new_posn); >> + sendUpdate(vm_sock, neg1, sizeof(long), s->shm_fd); >> + printf("[NC] trying to send fds to new connection\n"); >> + sendRights(vm_sock, new_posn, sizeof(new_posn), s->live_vms, >> s->msi_vectors); >> + >> + printf("[NC] Connected (count = %ld).\n", new_posn); >> + for (i = 0; i< new_posn; i++) { >> + if (s->live_vms[i].alive) { >> + // ping all clients that a new client has joined >> + printf("[UD] sending fd[%ld] to %ld\n", new_posn, i); >> + for (j = 0; j< s->msi_vectors; j++) { >> + printf("\tefd[%ld] = [%d]", j, >> s->live_vms[new_posn].efd[j]); >> + sendUpdate(s->live_vms[i].sockfd, new_posn, >> + sizeof(new_posn), s->live_vms[new_posn].efd[j]); >> + } >> + printf("\n"); >> + } >> + } >> + >> + s->total_count++; >> +} >> + >> +int create_listening_socket(char * path) { >> + >> + struct sockaddr_un local; >> + int len, conn_socket; >> + >> + if ((conn_socket = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { >> + perror("socket"); >> + exit(1); >> + } >> + >> + local.sun_family = AF_UNIX; >> + strcpy(local.sun_path, path); >> + unlink(local.sun_path); >> + len = strlen(local.sun_path) + sizeof(local.sun_family); >> + if (bind(conn_socket, (struct sockaddr *)&local, len) == -1) { >> + perror("bind"); >> + exit(1); >> + } >> + >> + if (listen(conn_socket, 5) == -1) { >> + perror("listen"); >> + exit(1); >> + } >> + >> + return conn_socket; >> + >> +} >> + >> +void parse_args(int argc, char **argv, server_state_t * s) { >> + >> + int c; >> + >> + s->shm_size = 1024 * 1024; // default shm_size >> + s->path = NULL; >> + s->shmobj = NULL; >> + s->msi_vectors = 1; >> + >> + while ((c = getopt(argc, argv, "hp:s:m:n:")) != -1) { >> + >> + switch (c) { >> + // path to listening socket >> + case 'p': >> + s->path = optarg; >> + break; >> + // name of shared memory object >> + case 's': >> + s->shmobj = optarg; >> + break; >> + // size of shared memory object >> + case 'm': { >> + uint64_t value; >> + char *ptr; >> + >> + value = strtoul(optarg,&ptr, 10); >> + switch (*ptr) { >> + case 0: case 'M': case 'm': >> + value<<= 20; >> + break; >> + case 'G': case 'g': >> + value<<= 30; >> + break; >> + default: >> + fprintf(stderr, "qemu: invalid ram size: %s\n", >> optarg); >> + exit(1); >> + } >> + s->shm_size = value; >> + break; >> + } >> + case 'n': >> + s->msi_vectors = atol(optarg); >> + break; >> + case 'h': >> + default: >> + usage(argv[0]); >> + exit(1); >> + } >> + } >> + >> + if (s->path == NULL) { >> + s->path = strdup(DEFAULT_SOCK_PATH); >> + } >> + >> + printf("listening socket: %s\n", s->path); >> + >> + if (s->shmobj == NULL) { >> + s->shmobj = strdup(DEFAULT_SHM_OBJ); >> + } >> + >> + printf("shared object: %s\n", s->shmobj); >> + printf("shared object size: %d (bytes)\n", s->shm_size); >> + >> +} >> + >> +void print_vec(server_state_t * s, const char * c) { >> + >> + int i, j; >> + >> +#if DEBUG >> + printf("%s (%ld) = ", c, s->total_count); >> + for (i = 0; i< s->total_count; i++) { >> + if (s->live_vms[i].alive) { >> + for (j = 0; j< s->msi_vectors; j++) { >> + printf("[%d|%d] ", s->live_vms[i].sockfd, >> s->live_vms[i].efd[j]); >> + } >> + } >> + } >> + printf("\n"); >> +#endif >> + >> +} >> + >> +int find_set(fd_set * readset, int max) { >> + >> + int i; >> + >> + for (i = 1; i< max; i++) { >> + if (FD_ISSET(i, readset)) { >> + return i; >> + } >> + } >> + >> + printf("nothing set\n"); >> + return -1; >> + >> +} >> + >> +void usage(char const *prg) { >> + fprintf(stderr, "use: %s [-h] [-p<unix socket>] [-s<shm obj>] " >> + "[-m<size in MB>] [-n<# of MSI vectors>]\n", prg); >> +} >> diff --git a/contrib/ivshmem-server/send_scm.c >> b/contrib/ivshmem-server/send_scm.c >> new file mode 100644 >> index 0000000..b1bb4a3 >> --- /dev/null >> +++ b/contrib/ivshmem-server/send_scm.c >> @@ -0,0 +1,208 @@ >> +#include<stdint.h> >> +#include<stdlib.h> >> +#include<errno.h> >> +#include<stdio.h> >> +#include<unistd.h> >> +#include<sys/socket.h> >> +#include<sys/syscall.h> >> +#include<sys/un.h> >> +#include<sys/types.h> >> +#include<sys/stat.h> >> +#include<fcntl.h> >> +#include<poll.h> >> +#include "send_scm.h" >> + >> +#ifndef POLLRDHUP >> +#define POLLRDHUP 0x2000 >> +#endif >> + >> +int readUpdate(int fd, long * posn, int * newfd) >> +{ >> + struct msghdr msg; >> + struct iovec iov[1]; >> + struct cmsghdr *cmptr; >> + size_t len; >> + size_t msg_size = sizeof(int); >> + char control[CMSG_SPACE(msg_size)]; >> + >> + msg.msg_name = 0; >> + msg.msg_namelen = 0; >> + msg.msg_control = control; >> + msg.msg_controllen = sizeof(control); >> + msg.msg_flags = 0; >> + msg.msg_iov = iov; >> + msg.msg_iovlen = 1; >> + >> + iov[0].iov_base =&posn; >> + iov[0].iov_len = sizeof(posn); >> + >> + do { >> + len = recvmsg(fd,&msg, 0); >> + } while (len == (size_t) (-1)&& (errno == EINTR || errno == >> EAGAIN)); >> + >> + printf("iov[0].buf is %ld\n", *((long *)iov[0].iov_base)); >> + printf("len is %ld\n", len); >> + // TODO: Logging >> + if (len == (size_t) (-1)) { >> + perror("recvmsg()"); >> + return -1; >> + } >> + >> + if (msg.msg_controllen< sizeof(struct cmsghdr)) >> + return *posn; >> + >> + for (cmptr = CMSG_FIRSTHDR(&msg); cmptr != NULL; >> + cmptr = CMSG_NXTHDR(&msg, cmptr)) { >> + if (cmptr->cmsg_level != SOL_SOCKET || >> + cmptr->cmsg_type != SCM_RIGHTS){ >> + printf("continuing %ld\n", sizeof(size_t)); >> + printf("read msg_size = %ld\n", msg_size); >> + if (cmptr->cmsg_len != sizeof(control)) >> + printf("not equal (%ld != >> %ld)\n",cmptr->cmsg_len,sizeof(control)); >> + continue; >> + } >> + >> + memcpy(newfd, CMSG_DATA(cmptr), sizeof(int)); >> + printf("posn is %ld (fd = %d)\n", *posn, *newfd); >> + return 0; >> + } >> + >> + fprintf(stderr, "bad data in packet\n"); >> + return -1; >> +} >> + >> +int readRights(int fd, long count, size_t count_len, int **fds, int >> msi_vectors) >> +{ >> + int j, newfd; >> + >> + for (; ;){ >> + long posn = 0; >> + >> + readUpdate(fd,&posn,&newfd); >> + printf("reading posn %ld ", posn); >> + fds[posn] = (int *)malloc (msi_vectors * sizeof(int)); >> + fds[posn][0] = newfd; >> + for (j = 1; j< msi_vectors; j++) { >> + readUpdate(fd,&posn,&newfd); >> + fds[posn][j] = newfd; >> + printf("%d.", fds[posn][j]); >> + } >> + printf("\n"); >> + >> + /* stop reading once i've read my own eventfds */ >> + if (posn == count) >> + break; >> + } >> + >> + return 0; >> +} >> + >> +int sendKill(int fd, long const posn, size_t posn_len) { >> + >> + struct cmsghdr *cmsg; >> + size_t msg_size = sizeof(int); >> + char control[CMSG_SPACE(msg_size)]; >> + struct iovec iov[1]; >> + size_t len; >> + struct msghdr msg = { 0, 0, iov, 1, control, sizeof control, 0 }; >> + >> + struct pollfd mypollfd; >> + int rv; >> + >> + iov[0].iov_base = (void *)&posn; >> + iov[0].iov_len = posn_len; >> + >> + // from cmsg(3) >> + cmsg = CMSG_FIRSTHDR(&msg); >> + cmsg->cmsg_level = SOL_SOCKET; >> + cmsg->cmsg_len = 0; >> + msg.msg_controllen = cmsg->cmsg_len; >> + >> + printf("Killing posn %ld\n", posn); >> + >> + // check if the fd is dead or not >> + mypollfd.fd = fd; >> + mypollfd.events = POLLRDHUP; >> + mypollfd.revents = 0; >> + >> + rv = poll(&mypollfd, 1, 0); >> + >> + printf("rv is %d\n", rv); >> + >> + if (rv == 0) { >> + len = sendmsg(fd,&msg, 0); >> + if (len == (size_t) (-1)) { >> + perror("sendmsg()"); >> + return -1; >> + } >> + return (len == posn_len); >> + } else { >> + printf("already dead\n"); >> + return 0; >> + } >> +} >> + >> +int sendUpdate(int fd, long posn, size_t posn_len, int sendfd) >> +{ >> + >> + struct cmsghdr *cmsg; >> + size_t msg_size = sizeof(int); >> + char control[CMSG_SPACE(msg_size)]; >> + struct iovec iov[1]; >> + size_t len; >> + struct msghdr msg = { 0, 0, iov, 1, control, sizeof control, 0 }; >> + >> + iov[0].iov_base = (void *) (&posn); >> + iov[0].iov_len = posn_len; >> + >> + // from cmsg(3) >> + cmsg = CMSG_FIRSTHDR(&msg); >> + cmsg->cmsg_level = SOL_SOCKET; >> + cmsg->cmsg_type = SCM_RIGHTS; >> + cmsg->cmsg_len = CMSG_LEN(msg_size); >> + msg.msg_controllen = cmsg->cmsg_len; >> + >> + memcpy((CMSG_DATA(cmsg)),&sendfd, msg_size); >> + >> + len = sendmsg(fd,&msg, 0); >> + if (len == (size_t) (-1)) { >> + perror("sendmsg()"); >> + return -1; >> + } >> + >> + return (len == posn_len); >> + >> +} >> + >> +int sendPosition(int fd, long const posn) >> +{ >> + int rv; >> + >> + rv = send(fd,&posn, sizeof(long), 0); >> + if (rv != sizeof(long)) { >> + fprintf(stderr, "error sending posn\n"); >> + return -1; >> + } >> + >> + return 0; >> +} >> + >> +int sendRights(int fd, long const count, size_t count_len, vmguest_t * >> Live_vms, >> + long >> msi_vectors) >> +{ >> + /* updates about new guests are sent one at a time */ >> + >> + long i, j; >> + >> + for (i = 0; i<= count; i++) { >> + if (Live_vms[i].alive) { >> + for (j = 0; j< msi_vectors; j++) { >> + sendUpdate(Live_vms[count].sockfd, i, sizeof(long), >> + >> Live_vms[i].efd[j]); >> + } >> + } >> + } >> + >> + return 0; >> + >> +} >> diff --git a/contrib/ivshmem-server/send_scm.h >> b/contrib/ivshmem-server/send_scm.h >> new file mode 100644 >> index 0000000..48c9a8d >> --- /dev/null >> +++ b/contrib/ivshmem-server/send_scm.h >> @@ -0,0 +1,19 @@ >> +#ifndef SEND_SCM >> +#define SEND_SCM >> + >> +struct vm_guest_conn { >> + int posn; >> + int sockfd; >> + int * efd; >> + int alive; >> +}; >> + >> +typedef struct vm_guest_conn vmguest_t; >> + >> +int readRights(int fd, long count, size_t count_len, int **fds, int >> msi_vectors); >> +int sendRights(int fd, long const count, size_t count_len, vmguest_t >> *Live_vms, long msi_vectors); >> +int readUpdate(int fd, long * posn, int * newfd); >> +int sendUpdate(int fd, long const posn, size_t posn_len, int sendfd); >> +int sendPosition(int fd, long const posn); >> +int sendKill(int fd, long const posn, size_t posn_len); >> +#endif >> > > -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html