On Tue, Dec 06, 2022 at 08:54:28PM +0000, Arseniy Krasnov wrote:
This adds utility to check vsock rx/tx performance.

Usage as sender:
./vsock_perf --sender <cid> --port <port> --bytes <bytes to send)

Little typo "<bytes to send>". I mean replace `)` with `>`

Usage as receiver:
./vsock_perf --port <port> --rcvlowat <SO_RCVLOWAT>

Signed-off-by: Arseniy Krasnov <avkras...@sberdevices.ru>
---
tools/testing/vsock/Makefile     |   3 +-
tools/testing/vsock/README       |  34 +++
tools/testing/vsock/vsock_perf.c | 434 +++++++++++++++++++++++++++++++
3 files changed, 470 insertions(+), 1 deletion(-)
create mode 100644 tools/testing/vsock/vsock_perf.c

diff --git a/tools/testing/vsock/Makefile b/tools/testing/vsock/Makefile
index f8293c6910c9..43a254f0e14d 100644
--- a/tools/testing/vsock/Makefile
+++ b/tools/testing/vsock/Makefile
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: GPL-2.0-only
-all: test
+all: test vsock_perf
test: vsock_test vsock_diag_test
vsock_test: vsock_test.o timeout.o control.o util.o
vsock_diag_test: vsock_diag_test.o timeout.o control.o util.o
+vsock_perf: vsock_perf.o

CFLAGS += -g -O2 -Werror -Wall -I. -I../../include -I../../../usr/include 
-Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD 
-U_FORTIFY_SOURCE -D_GNU_SOURCE
.PHONY: all test clean
diff --git a/tools/testing/vsock/README b/tools/testing/vsock/README
index 4d5045e7d2c3..e6f6735bba05 100644
--- a/tools/testing/vsock/README
+++ b/tools/testing/vsock/README
@@ -35,3 +35,37 @@ Invoke test binaries in both directions as follows:
                       --control-port=$GUEST_IP \
                       --control-port=1234 \
                       --peer-cid=3
+
+vsock_perf utility
+-------------------
+'vsock_perf' is a simple tool to measure vsock performance. It works in
+sender/receiver modes: sender connect to peer at the specified port and
+starts data transmission to the receiver. After data processing is done,
+it prints several metrics(see below).
+
+Usage:
+# run as sender
+# connect to CID 2, port 1234, send 1G of data, tx buf size is 1M
+./vsock_perf --sender 2 --port 1234 --bytes 1G --buf-size 1M
+
+Output:
+tx performance: A Gb/s
+
+Output explanation:
+A is calculated as "number of bytes to send" / "time in tx loop"
+
+# run as receiver
+# listen port 1234, rx buf size is 1M, socket buf size is 1G, SO_RCVLOWAT is 
64K
+./vsock_perf --port 1234 --buf-size 1M --vsk-size 1G --rcvlowat 64K
+
+Output:
+rx performance: A Gb/s
+total in 'read()': B sec
+POLLIN wakeups: C
+average in 'read()': D ns
+
+Output explanation:
+A is calculated as "number of received bytes" / "time in rx loop".
+B is time, spent in 'read()' system call(excluding 'poll()')
+C is number of 'poll()' wake ups with POLLIN bit set.
+D is B / C, e.g. average amount of time, spent in single 'read()'.
diff --git a/tools/testing/vsock/vsock_perf.c b/tools/testing/vsock/vsock_perf.c
new file mode 100644
index 000000000000..248fc3285d5c
--- /dev/null
+++ b/tools/testing/vsock/vsock_perf.c
@@ -0,0 +1,434 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * vsock_perf - benchmark utility for vsock.
+ *
+ * Copyright (C) 2022 SberDevices.
+ *
+ * Author: Arseniy Krasnov <avkras...@sberdevices.ru>
+ */
+#include <getopt.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <time.h>
+#include <stdint.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <linux/vm_sockets.h>
+
+#define DEFAULT_BUF_SIZE_BYTES (128 * 1024)
+#define DEFAULT_TO_SEND_BYTES  (64 * 1024)
+#define DEFAULT_VSOCK_BUF_BYTES (256 * 1024)
+#define DEFAULT_RCVLOWAT_BYTES 1
+#define DEFAULT_PORT           1234
+
+#define BYTES_PER_GB           (1024 * 1024 * 1024ULL)
+#define NSEC_PER_SEC           (1000000000ULL)
+
+static unsigned int port = DEFAULT_PORT;
+static unsigned long buf_size_bytes = DEFAULT_BUF_SIZE_BYTES;
+static unsigned long vsock_buf_bytes = DEFAULT_VSOCK_BUF_BYTES;
+
+static inline time_t current_nsec(void)
+{
+       struct timespec ts;
+
+       if (clock_gettime(CLOCK_REALTIME, &ts)) {
+               perror("clock_gettime");
+               exit(EXIT_FAILURE);
+       }
+
+       return (ts.tv_sec * NSEC_PER_SEC) + ts.tv_nsec;
+}
+
+/* From lib/cmdline.c. */
+static unsigned long memparse(const char *ptr)
+{
+       char *endptr;
+
+       unsigned long long ret = strtoull(ptr, &endptr, 0);
+
+       switch (*endptr) {
+       case 'E':
+       case 'e':
+               ret <<= 10;
+       case 'P':
+       case 'p':
+               ret <<= 10;
+       case 'T':
+       case 't':
+               ret <<= 10;
+       case 'G':
+       case 'g':
+               ret <<= 10;
+       case 'M':
+       case 'm':
+               ret <<= 10;
+       case 'K':
+       case 'k':
+               ret <<= 10;
+               endptr++;
+       default:
+               break;
+       }
+
+       return ret;
+}
+
+static void vsock_increase_buf_size(int fd)
+{
+       if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_MAX_SIZE,
+                      &vsock_buf_bytes, sizeof(vsock_buf_bytes))) {
+               perror("setsockopt(SO_VM_SOCKETS_BUFFER_MAX_SIZE)");
+               exit(EXIT_FAILURE);
+       }
+
+       if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_SIZE,
+                      &vsock_buf_bytes, sizeof(vsock_buf_bytes))) {
+               perror("setsockopt(SO_VM_SOCKETS_BUFFER_SIZE)");
+               exit(EXIT_FAILURE);
+       }
+}
+
+static int vsock_connect(unsigned int cid, unsigned int port)
+{
+       union {
+               struct sockaddr sa;
+               struct sockaddr_vm svm;
+       } addr = {
+               .svm = {
+                       .svm_family = AF_VSOCK,
+                       .svm_port = port,
+                       .svm_cid = cid,
+               },
+       };
+       int fd;
+
+       fd = socket(AF_VSOCK, SOCK_STREAM, 0);
+
+       if (fd < 0) {
+               perror("socket");
+               return -1;
+       }
+
+       if (connect(fd, &addr.sa, sizeof(addr.svm)) < 0) {
+               perror("connect");
+               close(fd);
+               return -1;
+       }
+
+       return fd;
+}
+
+static float get_gbps(unsigned long bytes, time_t ns_delta)
+{
+       return ((float)bytes / BYTES_PER_GB) /
+              ((float)ns_delta / NSEC_PER_SEC);
+}
+
+static void run_receiver(unsigned long rcvlowat_bytes)
+{
+       unsigned int read_cnt;
+       time_t rx_begin_ns;
+       time_t in_read_ns;
+       size_t total_recv;
+       int client_fd;
+       char *data;
+       int fd;
+       union {
+               struct sockaddr sa;
+               struct sockaddr_vm svm;
+       } addr = {
+               .svm = {
+                       .svm_family = AF_VSOCK,
+                       .svm_port = port,
+                       .svm_cid = VMADDR_CID_ANY,
+               },
+       };
+       union {
+               struct sockaddr sa;
+               struct sockaddr_vm svm;
+       } clientaddr;
+
+       socklen_t clientaddr_len = sizeof(clientaddr.svm);
+
+       printf("Run as receiver\n");
+       printf("Listen port %u\n", port);
+       printf("RX buffer %lu bytes\n", buf_size_bytes);
+       printf("vsock buffer %lu bytes\n", vsock_buf_bytes);
+       printf("SO_RCVLOWAT %lu bytes\n", rcvlowat_bytes);
+
+       fd = socket(AF_VSOCK, SOCK_STREAM, 0);
+
+       if (fd < 0) {
+               perror("socket");
+               exit(EXIT_FAILURE);
+       }
+
+       if (bind(fd, &addr.sa, sizeof(addr.svm)) < 0) {
+               perror("bind");
+               exit(EXIT_FAILURE);
+       }
+
+       if (listen(fd, 1) < 0) {
+               perror("listen");
+               exit(EXIT_FAILURE);
+       }
+
+       client_fd = accept(fd, &clientaddr.sa, &clientaddr_len);
+
+       if (client_fd < 0) {
+               perror("accept");
+               exit(EXIT_FAILURE);
+       }
+
+       vsock_increase_buf_size(client_fd);
+
+       if (setsockopt(client_fd, SOL_SOCKET, SO_RCVLOWAT,
+                      &rcvlowat_bytes,
+                      sizeof(rcvlowat_bytes))) {
+               perror("setsockopt(SO_RCVLOWAT)");
+               exit(EXIT_FAILURE);
+       }
+
+       data = malloc(buf_size_bytes);
+
+       if (!data) {
+               printf("malloc failed\n");

This is an error message, so we can use fprintf(stderr, ...)

+               exit(EXIT_FAILURE);
+       }
+
+       read_cnt = 0;
+       in_read_ns = 0;
+       total_recv = 0;
+       rx_begin_ns = current_nsec();
+
+       while (1) {
+               struct pollfd fds = { 0 };
+
+               fds.fd = client_fd;
+               fds.events = POLLIN | POLLERR |
+                            POLLHUP | POLLRDHUP;
+
+               if (poll(&fds, 1, -1) < 0) {
+                       perror("poll");
+                       exit(EXIT_FAILURE);
+               }
+
+               if (fds.revents & POLLERR) {
+                       printf("'poll()' error\n");

Ditto.

+                       exit(EXIT_FAILURE);
+               }
+
+               if (fds.revents & POLLIN) {
+                       ssize_t bytes_read;
+                       time_t t;
+
+                       t = current_nsec();
+                       bytes_read = read(fds.fd, data, buf_size_bytes);
+                       in_read_ns += (current_nsec() - t);
+                       read_cnt++;
+
+                       if (!bytes_read)
+                               break;
+
+                       if (bytes_read < 0) {
+                               perror("read");
+                               exit(EXIT_FAILURE);
+                       }
+
+                       total_recv += bytes_read;
+               }
+
+               if (fds.revents & (POLLHUP | POLLRDHUP))
+                       break;
+       }
+
+       printf("rx performance: %f Gb/s\n",
+              get_gbps(total_recv, current_nsec() - rx_begin_ns));
+       printf("total time in 'read()': %f sec\n", (float)in_read_ns / 
NSEC_PER_SEC);
+       printf("POLLIN wakeups: %i\n", read_cnt);
+       printf("average time in 'read()': %f ns\n", (float)in_read_ns / 
read_cnt);
+
+       free(data);
+       close(client_fd);
+       close(fd);
+}
+
+static void run_sender(int peer_cid, unsigned long to_send_bytes)
+{
+       time_t tx_begin_ns;
+       size_t total_send;
+       void *data;
+       int fd;
+
+       printf("Run as sender\n");
+       printf("Connect to %i:%u\n", peer_cid, port);
+       printf("Send %lu bytes\n", to_send_bytes);
+       printf("TX buffer %lu bytes\n", buf_size_bytes);
+
+       fd = vsock_connect(peer_cid, port);
+
+       if (fd < 0)
+               exit(EXIT_FAILURE);
+
+       data = malloc(buf_size_bytes);
+
+       if (!data) {
+               printf("malloc failed\n");

Ditto.

+               exit(EXIT_FAILURE);
+       }
+
+       memset(data, 0, buf_size_bytes);
+       total_send = 0;
+       tx_begin_ns = current_nsec();
+
+       while (total_send < to_send_bytes) {
+               ssize_t sent;
+
+               sent = write(fd, data, buf_size_bytes);
+
+               if (sent <= 0) {
+                       perror("write");
+                       exit(EXIT_FAILURE);
+               }
+
+               total_send += sent;
+       }

I don't know if it would be useful, but maybe we can also print the total number of bytes and the time spent (the same in the receiver as well).

Here I would add

        tx_end_ns = current_nsec();

Then use it later.

+
+       printf("tx performance: %f Gb/s\n",
+              get_gbps(total_send, current_nsec() - tx_begin_ns));
+

Did you compared the performance with iperf-vsock and uperf?

I see an order of magnitude difference but I did not have time to check all the parameters to make a fair comparison. It would be helpful if you can compare and report it.

Thanks,
Stefano

+       close(fd);
+       free(data);
+}
+
+static const char optstring[] = "";
+static const struct option longopts[] = {
+       {
+               .name = "help",
+               .has_arg = no_argument,
+               .val = 'H',
+       },
+       {
+               .name = "sender",
+               .has_arg = required_argument,
+               .val = 'S',
+       },
+       {
+               .name = "port",
+               .has_arg = required_argument,
+               .val = 'P',
+       },
+       {
+               .name = "bytes",
+               .has_arg = required_argument,
+               .val = 'M',
+       },
+       {
+               .name = "buf-size",
+               .has_arg = required_argument,
+               .val = 'B',
+       },
+       {
+               .name = "vsk-size",
+               .has_arg = required_argument,
+               .val = 'V',
+       },
+       {
+               .name = "rcvlowat",
+               .has_arg = required_argument,
+               .val = 'R',
+       },
+       {},
+};
+
+static void usage(void)
+{
+       printf("Usage: ./vsock_perf [--help] [options]\n"
+              "\n"
+              "This is benchmarking utility, to test vsock performance.\n"
+              "It runs in two modes: sender or receiver. In sender mode, it\n"
+              "connects to the specified CID and starts data transmission.\n"
+              "\n"
+              "Options:\n"
+              "  --help                   This message\n"
+              "  --sender   <cid>           Sender mode (receiver default)\n"
+              "                                <cid> of the receiver to connect 
to\n"
+              "  --port     <port>          Port (default %d)\n"
+              "  --bytes    <bytes>KMG              Bytes to send (default 
%d)\n"
+              "  --buf-size <bytes>KMG              Data buffer size (default %d). 
In sender mode\n"
+              "                                it is the buffer size, passed to 
'write()'. In\n"
+              "                                receiver mode it is the buffer size 
passed to 'read()'.\n"
+              "  --vsk-size <bytes>KMG              Socket buffer size (default 
%d)\n"
+              "  --rcvlowat <bytes>KMG              SO_RCVLOWAT value (default 
%d)\n"
+              "\n", DEFAULT_PORT, DEFAULT_TO_SEND_BYTES,
+              DEFAULT_BUF_SIZE_BYTES, DEFAULT_VSOCK_BUF_BYTES,
+              DEFAULT_RCVLOWAT_BYTES);
+       exit(EXIT_FAILURE);
+}
+
+static long strtolx(const char *arg)
+{
+       long value;
+       char *end;
+
+       value = strtol(arg, &end, 10);
+
+       if (end != arg + strlen(arg))
+               usage();
+
+       return value;
+}
+
+int main(int argc, char **argv)
+{
+       unsigned long to_send_bytes = DEFAULT_TO_SEND_BYTES;
+       unsigned long rcvlowat_bytes = DEFAULT_RCVLOWAT_BYTES;
+       int peer_cid = -1;
+       bool sender = false;
+
+       while (1) {
+               int opt = getopt_long(argc, argv, optstring, longopts, NULL);
+
+               if (opt == -1)
+                       break;
+
+               switch (opt) {
+               case 'V': /* Peer buffer size. */
+                       vsock_buf_bytes = memparse(optarg);
+                       break;
+               case 'R': /* SO_RCVLOWAT value. */
+                       rcvlowat_bytes = memparse(optarg);
+                       break;
+               case 'P': /* Port to connect to. */
+                       port = strtolx(optarg);
+                       break;
+               case 'M': /* Bytes to send. */
+                       to_send_bytes = memparse(optarg);
+                       break;
+               case 'B': /* Size of rx/tx buffer. */
+                       buf_size_bytes = memparse(optarg);
+                       break;
+               case 'S': /* Sender mode. CID to connect to. */
+                       peer_cid = strtolx(optarg);
+                       sender = true;
+                       break;
+               case 'H': /* Help. */
+                       usage();
+                       break;
+               default:
+                       usage();
+               }
+       }
+
+       if (!sender)
+               run_receiver(rcvlowat_bytes);
+       else
+               run_sender(peer_cid, to_send_bytes);
+
+       return 0;
+}
--
2.25.1

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Reply via email to