# HG changeset patch # User Roman Arutyunyan <a...@nginx.com> # Date 1690460906 -14400 # Thu Jul 27 16:28:26 2023 +0400 # Node ID 4f078be6e2ed08643371a3956f5f18f2357a38db # Parent d8a6ec55938e9a4f1a84c825d9c6abe13aa8b791 QUIC: use sendmmsg() with GSO.
This syscall appeared before GSO and allows to send more datagrams with a single call. diff --git a/auto/os/linux b/auto/os/linux --- a/auto/os/linux +++ b/auto/os/linux @@ -291,4 +291,16 @@ ngx_feature_test="socklen_t optlen = siz . auto/feature +ngx_feature="sendmmsg()" +ngx_feature_name="NGX_HAVE_SENDMMSG" +ngx_feature_run=no +ngx_feature_incs="#include <sys/socket.h> + #include <sys/uio.h>" +ngx_feature_path= +ngx_feature_libs= +ngx_feature_test="struct mmsghdr msg[UIO_MAXIOV]; + sendmmsg(0, msg, UIO_MAXIOV, 0);" +. auto/feature + + CC_AUX_FLAGS="$cc_aux_flags -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64" diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -42,11 +42,9 @@ static ngx_int_t ngx_quic_create_datagra static void ngx_quic_commit_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx); static void ngx_quic_revert_send(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, uint64_t pnum); -#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) +#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG) static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c); static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c); -static ssize_t ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, - size_t len, struct sockaddr *sockaddr, socklen_t socklen, size_t segment); #endif static ssize_t ngx_quic_output_packet(ngx_connection_t *c, ngx_quic_send_ctx_t *ctx, u_char *data, size_t max, size_t min); @@ -91,7 +89,7 @@ ngx_quic_output(ngx_connection_t *c) in_flight = cg->in_flight; -#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) +#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG) if (ngx_quic_allow_segmentation(c)) { rc = ngx_quic_create_segments(c); } else @@ -264,7 +262,7 @@ ngx_quic_revert_send(ngx_connection_t *c } -#if ((NGX_HAVE_UDP_SEGMENT) && (NGX_HAVE_MSGHDR_MSG_CONTROL)) +#if (NGX_HAVE_UDP_SEGMENT && NGX_HAVE_MSGHDR_MSG_CONTROL && NGX_HAVE_SENDMMSG) static ngx_uint_t ngx_quic_allow_segmentation(ngx_connection_t *c) @@ -324,16 +322,28 @@ ngx_quic_allow_segmentation(ngx_connecti static ngx_int_t ngx_quic_create_segments(ngx_connection_t *c) { - size_t len, segsize; + size_t len, segsize, clen; ssize_t n; - u_char *p, *end; + u_char *p, *start, *end; + uint16_t *valp; uint64_t preserved_pnum; - ngx_uint_t nseg; + ngx_err_t err; + ngx_uint_t nseg, nmsg; + struct msghdr msg; + struct cmsghdr *cmsg; ngx_quic_path_t *path; ngx_quic_send_ctx_t *ctx; ngx_quic_congestion_t *cg; ngx_quic_connection_t *qc; - static u_char dst[NGX_QUIC_MAX_UDP_SEGMENT_BUF]; +#if (NGX_HAVE_ADDRINFO_CMSG) + char msg_control[CMSG_SPACE(sizeof(uint16_t)) + + CMSG_SPACE(sizeof(ngx_addrinfo_t))]; +#else + char msg_control[CMSG_SPACE(sizeof(uint16_t))]; +#endif + struct iovec iovs[UIO_MAXIOV]; + static u_char dst[UIO_MAXIOV * NGX_QUIC_MAX_UDP_SEGMENT_BUF]; + struct mmsghdr msgs[UIO_MAXIOV]; qc = ngx_quic_get_connection(c); cg = &qc->congestion; @@ -347,94 +357,12 @@ ngx_quic_create_segments(ngx_connection_ segsize = ngx_min(qc->ctp.max_udp_payload_size, NGX_QUIC_MAX_UDP_SEGMENT_BUF); - p = dst; - end = dst + sizeof(dst); - - nseg = 0; - - preserved_pnum = ctx->pnum; - - for ( ;; ) { - - len = ngx_min(segsize, (size_t) (end - p)); - - if (len && cg->in_flight + (p - dst) < cg->window) { - - n = ngx_quic_output_packet(c, ctx, p, len, len); - if (n == NGX_ERROR) { - return NGX_ERROR; - } - - if (n) { - p += n; - nseg++; - } - - } else { - n = 0; - } - - if (p == dst) { - break; - } - - if (n == 0 || nseg == NGX_QUIC_MAX_SEGMENTS) { - n = ngx_quic_send_segments(c, dst, p - dst, path->sockaddr, - path->socklen, segsize); - if (n == NGX_ERROR) { - return NGX_ERROR; - } - - if (n == NGX_AGAIN) { - ngx_quic_revert_send(c, ctx, preserved_pnum); - - ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY); - break; - } - - ngx_quic_commit_send(c, ctx); - - path->sent += n; - - p = dst; - nseg = 0; - preserved_pnum = ctx->pnum; - } - } - - return NGX_OK; -} - - -static ssize_t -ngx_quic_send_segments(ngx_connection_t *c, u_char *buf, size_t len, - struct sockaddr *sockaddr, socklen_t socklen, size_t segment) -{ - size_t clen; - ssize_t n; - uint16_t *valp; - struct iovec iov; - struct msghdr msg; - struct cmsghdr *cmsg; - -#if (NGX_HAVE_ADDRINFO_CMSG) - char msg_control[CMSG_SPACE(sizeof(uint16_t)) - + CMSG_SPACE(sizeof(ngx_addrinfo_t))]; -#else - char msg_control[CMSG_SPACE(sizeof(uint16_t))]; -#endif ngx_memzero(&msg, sizeof(struct msghdr)); ngx_memzero(msg_control, sizeof(msg_control)); - iov.iov_len = len; - iov.iov_base = buf; - - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - - msg.msg_name = sockaddr; - msg.msg_namelen = socklen; + msg.msg_name = path->sockaddr; + msg.msg_namelen = path->socklen; msg.msg_control = msg_control; msg.msg_controllen = sizeof(msg_control); @@ -448,7 +376,7 @@ ngx_quic_send_segments(ngx_connection_t clen = CMSG_SPACE(sizeof(uint16_t)); valp = (void *) CMSG_DATA(cmsg); - *valp = segment; + *valp = segsize; #if (NGX_HAVE_ADDRINFO_CMSG) if (c->listening && c->listening->wildcard && c->local_sockaddr) { @@ -459,14 +387,100 @@ ngx_quic_send_segments(ngx_connection_t msg.msg_controllen = clen; - n = ngx_sendmsg(c, &msg, 0); - if (n < 0) { - return n; + for ( ;; ) { + + preserved_pnum = ctx->pnum; + + p = dst; + + for (nmsg = 0; nmsg < UIO_MAXIOV; nmsg++) { + + start = p; + end = p + NGX_QUIC_MAX_UDP_SEGMENT_BUF; + + for (nseg = 0; nseg < NGX_QUIC_MAX_SEGMENTS; nseg++) { + + if (cg->in_flight + (p - dst) >= cg->window) { + break; + } + + len = ngx_min(segsize, (size_t) (end - p)); + if (len == 0) { + break; + } + + n = ngx_quic_output_packet(c, ctx, p, len, len); + + if (n == NGX_ERROR) { + return NGX_ERROR; + } + + if (n == 0) { + break; + } + + p += n; + } + + if (nseg == 0) { + break; + } + + iovs[nmsg].iov_base = start; + iovs[nmsg].iov_len = p - start; + + msgs[nmsg].msg_hdr = msg; + msgs[nmsg].msg_hdr.msg_iov = &iovs[nmsg]; + msgs[nmsg].msg_hdr.msg_iovlen = 1; + msgs[nmsg].msg_len = 0; + } + + if (nmsg == 0) { + break; + } + + eintr: + + n = sendmmsg(c->fd, msgs, nmsg, 0); + + if (n == -1) { + err = ngx_errno; + + switch (err) { + case NGX_EAGAIN: + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, + "sendmmsg() not ready"); + + ngx_quic_revert_send(c, ctx, preserved_pnum); + + ngx_add_timer(&qc->push, NGX_QUIC_SOCKET_RETRY_DELAY); + return NGX_OK; + + case NGX_EINTR: + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, + "sendmmsg() was interrupted"); + goto eintr; + + default: + c->write->error = 1; + ngx_connection_error(c, err, "sendmsg() failed"); + return NGX_ERROR; + } + } + + len = p - dst; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, + "sendmmsg: %z of %ui msg of size %uz", n, nmsg, len); + + c->sent += len; + + ngx_quic_commit_send(c, ctx); + + path->sent += len; } - c->sent += n; - - return n; + return NGX_OK; } #endif _______________________________________________ nginx-devel mailing list nginx-devel@nginx.org https://mailman.nginx.org/mailman/listinfo/nginx-devel