On Thu, Feb 25, 2021 at 05:03:19PM +0100, Claudio Jeker wrote: > On Fri, Feb 19, 2021 at 07:10:02PM +0100, Claudio Jeker wrote: > > Some TAL files now include an https URI where the TA can be fetched from. > > With this diff rpki-client will download the TA from https unless that > > fails and then fall back to rsync. > > > > This is not yet perfect but the diff is already large enough (adding a > > full event based https client based on ftp codebase). For RRDP more a lot > > more is required and I probably will refactor the main.c code then. > > > > At the moment this adds a local mkostempat() function to implement > > mkostemp() but with openat() instead of open(). I hope in the future libc > > will provide something. > > > > Thanks in advance for all the feedback > > Updated diff. I found some bugs in the http.c code base regarding > conncetion failures (because of unreachable IPv6 on the server) and fixed > redirects. Those should now be fixed.
A couple of comments inline. Some cosmetic, some minor bugs. Apart from a return value glitch for tls_close() the libtls handling looks correct to me and while I'm not a http expert, the state machine makes sense. The string handling (chopping up and re-encoding) looks correct with appropriate bounds checking. I spent quite a bit of time going through it. Take from my comments whatever you like/makes sense, but I'm ok with landing this and improving in tree. > > -- > :wq Claudio > > Index: Makefile > =================================================================== > RCS file: /cvs/src/usr.sbin/rpki-client/Makefile,v > retrieving revision 1.18 > diff -u -p -r1.18 Makefile > --- Makefile 4 Feb 2021 08:10:24 -0000 1.18 > +++ Makefile 19 Feb 2021 14:21:08 -0000 > @@ -1,13 +1,14 @@ > # $OpenBSD: Makefile,v 1.18 2021/02/04 08:10:24 claudio Exp $ > > PROG= rpki-client > -SRCS= as.c cert.c cms.c crl.c gbr.c io.c ip.c log.c main.c mft.c > mkdir.c \ > - output.c output-bgpd.c output-bird.c output-csv.c output-json.c \ > - parser.c roa.c rsync.c tal.c validate.c x509.c > +SRCS= as.c cert.c cms.c crl.c gbr.c http.c io.c ip.c log.c main.c > mft.c \ > + mkdir.c mkostempat.c output.c output-bgpd.c output-bird.c \ > + output-csv.c output-json.c parser.c roa.c rsync.c tal.c validate.c \ > + x509.c > MAN= rpki-client.8 > > -LDADD+= -lcrypto -lutil > -DPADD+= ${LIBCRYPTO} ${LIBUTIL} > +LDADD+= -ltls -lssl -lcrypto -lutil > +DPADD+= ${LIBTLS} ${LIBSSL} ${LIBCRYPTO} ${LIBUTIL} > > CFLAGS+= -Wall -I${.CURDIR} > CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes > Index: extern.h > =================================================================== > RCS file: /cvs/src/usr.sbin/rpki-client/extern.h,v > retrieving revision 1.47 > diff -u -p -r1.47 extern.h > --- extern.h 22 Feb 2021 09:46:05 -0000 1.47 > +++ extern.h 23 Feb 2021 10:36:38 -0000 > @@ -399,6 +399,10 @@ void proc_parser(int) __attribute__((n > char *rsync_base_uri(const char *); > void proc_rsync(char *, char *, int) __attribute__((noreturn)); > > +/* Http-specific. */ > + > +void proc_http(char *, int); > + > /* Logging (though really used for OpenSSL errors). */ > > void cryptowarnx(const char *, ...) > @@ -417,6 +421,7 @@ void io_str_buffer(struct ibuf *, cons > void io_simple_read(int, void *, size_t); > void io_buf_read_alloc(int, void **, size_t *); > void io_str_read(int, char **); > +int io_recvfd(int, void *, size_t); > > /* X509 helpers. */ > > @@ -450,6 +455,7 @@ void logx(const char *fmt, ...) > __attribute__((format(printf, 1, 2))); > > int mkpathat(int, const char *); > +int mkostempat(int, char *, int); > > #define RPKI_PATH_OUT_DIR "/var/db/rpki-client" > #define RPKI_PATH_BASE_DIR "/var/cache/rpki-client" > Index: http.c > =================================================================== > RCS file: http.c > diff -N http.c > --- /dev/null 1 Jan 1970 00:00:00 -0000 > +++ http.c 24 Feb 2021 20:03:25 -0000 > @@ -0,0 +1,1226 @@ > +/* $OpenBSD$ */ > +/* > + * Copyright (c) 2020 Nils Fisher <nils_fis...@hotmail.com> > + * Copyright (c) 2020 Claudio Jeker <clau...@openbsd.com> > + * > + * Permission to use, copy, modify, and distribute this software for any > + * purpose with or without fee is hereby granted, provided that the above > + * copyright notice and this permission notice appear in all copies. > + * > + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + */ > + > +/*- > + * Copyright (c) 1997 The NetBSD Foundation, Inc. > + * All rights reserved. > + * > + * This code is derived from software contributed to The NetBSD Foundation > + * by Jason Thorpe and Luke Mewburn. > + * > + * Redistribution and use in source and binary forms, with or without > + * modification, are permitted provided that the following conditions > + * are met: > + * 1. Redistributions of source code must retain the above copyright > + * notice, this list of conditions and the following disclaimer. > + * 2. Redistributions in binary form must reproduce the above copyright > + * notice, this list of conditions and the following disclaimer in the > + * documentation and/or other materials provided with the distribution. > + * > + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS > + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > LIMITED > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR > + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS > + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR > + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS > + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN > + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) > + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE > + * POSSIBILITY OF SUCH DAMAGE. > + */ > +#include <sys/types.h> > +#include <sys/queue.h> > +#include <sys/socket.h> > + > +#include <ctype.h> > +#include <err.h> > +#include <errno.h> > +#include <limits.h> > +#include <netdb.h> > +#include <poll.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <unistd.h> > +#include <vis.h> > +#include <imsg.h> > + > +#include <tls.h> > + > +#include "extern.h" > + > +#define HTTP_USER_AGENT "OpenBSD rpki-client" > +#define HTTP_BUF_SIZE (32 * 1024) > +#define MAX_CONNECTIONS 12 > + > +#define WANT_POLLIN 1 > +#define WANT_POLLOUT 2 > + > +enum http_state { > + STATE_FREE, > + STATE_INIT, > + STATE_CONNECT, > + STATE_TLSCONNECT, > + STATE_REQUEST, > + STATE_RESPONSE_STATUS, > + STATE_RESPONSE_HEADER, > + STATE_RESPONSE_DATA, > + STATE_RESPONSE_CHUNKED, > + STATE_WRITE_DATA, > + STATE_DONE, > +}; > + > +struct http_proxy { > + char *proxyhost; > + char *proxyuser; > + char *proxypw; > +}; > + > +struct http_connection { > + char *url; > + char *host; > + char *port; > + char *path; /* points into url */ maybe make path const so it's clear that it's not meant for freeing? > + char *modified_since; > + char *last_modified; > + struct addrinfo *res0; > + struct addrinfo *res; > + struct tls *tls; > + char *buf; > + size_t bufsz; > + size_t bufpos; > + size_t id; > + size_t chunksz; > + off_t filesize; > + off_t bytes; > + int status; > + int redirect_loop; > + int fd; > + int outfd; > + short events; > + short chunked; > + enum http_state state; > +}; > + > +struct msgbuf msgq; > +struct sockaddr_storage http_bindaddr; > +struct tls_config *tls_config; > +uint8_t *tls_ca_mem; > +size_t tls_ca_size; > + > +char *resp_buf[MAX_CONNECTIONS]; > +size_t resp_bsz[MAX_CONNECTIONS]; > +size_t resp_idx; > + > +/* > + * Return a string that can be used in error message to identify the > + * connection. > + */ > +static const char * > +http_info(const char *url) > +{ > + static char buf[64]; > + > + if (strnvis(buf, url, sizeof buf, VIS_SAFE) >= (int)sizeof buf) { > + /* overflow, add indicator */ > + memcpy(buf + sizeof buf - 4, "...", 4); > + } > + > + return buf; > +} > + > +/* > + * Determine whether the character needs encoding, per RFC1738: > + * - No corresponding graphic US-ASCII. > + * - Unsafe characters. > + */ > +static int > +unsafe_char(const char *c0) > +{ > + const char *unsafe_chars = " <>\"#{}|\\^~[]`"; > + const unsigned char *c = (const unsigned char *)c0; > + > + /* > + * No corresponding graphic US-ASCII. > + * Control characters and octets not used in US-ASCII. > + */ > + return (iscntrl(*c) || !isascii(*c) || > + > + /* > + * Unsafe characters. > + * '%' is also unsafe, if is not followed by two > + * hexadecimal digits. > + */ > + strchr(unsafe_chars, *c) != NULL || > + (*c == '%' && (!isxdigit(*++c) || !isxdigit(*++c)))); > +} > + > +/* > + * Encode given URL, per RFC1738. > + * Allocate and return string to the caller. > + */ > +static char * > +url_encode(const char *path) > +{ > + size_t i, length, new_length; > + char *epath, *epathp; > + > + length = new_length = strlen(path); > + > + /* > + * First pass: > + * Count unsafe characters, and determine length of the > + * final URL. > + */ > + for (i = 0; i < length; i++) > + if (unsafe_char(path + i)) > + new_length += 2; > + > + epath = epathp = malloc(new_length + 1); /* One more for '\0'. */ > + if (epath == NULL) > + err(1, NULL); > + > + /* > + * Second pass: > + * Encode, and copy final URL. > + */ > + for (i = 0; i < length; i++) > + if (unsafe_char(path + i)) { > + snprintf(epathp, 4, "%%" "%02x", > + (unsigned char)path[i]); > + epathp += 3; > + } else > + *(epathp++) = path[i]; > + > + *epathp = '\0'; > + return (epath); > +} > + > +static void > +http_setup(void) > +{ > + tls_config = tls_config_new(); > + if (tls_config == NULL) > + errx(1, "tls config failed"); > +#if 0 > + /* TODO Should we allow extra protos and ciphers? */ > + if (tls_config_set_protocols(tls_config, > + TLS_PROTOCOLS_ALL) != 0) libtls returns -1 or NULL on error, so I would check for -1. I won't annotate this further. > + errx(1, "tls set protocols failed: %s", > + tls_config_error(tls_config)); > + if (tls_config_set_ciphers(tls_config, "legacy") != 0) > + errx(1, "tls set ciphers failed: %s", > + tls_config_error(tls_config)); > +#endif > + > + /* load cert file from disk now */ > + tls_ca_mem = tls_load_file(tls_default_ca_cert_file(), > + &tls_ca_size, NULL); > + if (tls_ca_mem == NULL) > + err(1, "tls_load_file"); > + tls_config_set_ca_mem(tls_config, tls_ca_mem, tls_ca_size); > + > + /* TODO initalize proxy settings */ > + > +} > + > +static int > +http_resolv(struct http_connection *conn, const char *host, const char *port) > +{ > + struct addrinfo hints; > + int error; > + > + memset(&hints, 0, sizeof(hints)); > + hints.ai_family = PF_UNSPEC; > + hints.ai_socktype = SOCK_STREAM; > + error = getaddrinfo(host, port, &hints, &conn->res0); > + /* > + * If the services file is corrupt/missing, fall back > + * on our hard-coded defines. > + */ > + if (error == EAI_SERVICE) > + error = getaddrinfo(host, "443", &hints, &conn->res0); > + if (error) { error != NULL > + warnx("%s: %s", host, gai_strerror(error)); > + return -1; > + } > + > + return 0; > +} > + > +static void > +http_done(struct http_connection *conn, int ok) > +{ > + struct ibuf *b; > + > + conn->state = STATE_DONE; > + > + if ((b = ibuf_dynamic(64, UINT_MAX)) == NULL) > + err(1, NULL); > + io_simple_buffer(b, &conn->id, sizeof(conn->id)); > + io_simple_buffer(b, &ok, sizeof(ok)); > +#if 0 /* TODO: cache last_modified */ > + io_str_buffer(b, conn->last_modified); > +#endif > + ibuf_close(&msgq, b); > +} > + > +static void > +http_fail(size_t id) > +{ > + struct ibuf *b; > + int ok = 0; > + > + if ((b = ibuf_dynamic(8, UINT_MAX)) == NULL) > + err(1, NULL); > + io_simple_buffer(b, &id, sizeof(id)); > + io_simple_buffer(b, &ok, sizeof(ok)); > + ibuf_close(&msgq, b); > +} > + > +static void > +http_free(struct http_connection *conn) > +{ > + if (conn->state != STATE_DONE) > + http_fail(conn->id); > + > + free(conn->host); > + free(conn->port); > + free(conn->url); I would move free(conn->url) to the top so you free stuff in the order of the struct members. > + /* no need to free conn->path it points into conn->uri */ typo: conn->url > + free(conn->modified_since); > + free(conn->last_modified); > + free(conn->buf); > + > + if (conn->res0) > + freeaddrinfo(conn->res0); > + > + tls_free(conn->tls); > + conn->tls = NULL; No need for setting this to NULL. > + > + if (conn->fd != -1) > + close(conn->fd); > + close(conn->outfd); > + free(conn); > +} > + > +static int > +http_close(struct http_connection *conn) > +{ > + if (conn->tls) { The file is a bit inconsistent with comparisons to NULL. I would prefer if they were done all explicitly but will stop annotating this here. > + switch (tls_close(conn->tls)) { > + case TLS_WANT_POLLIN: > + return WANT_POLLIN; > + case TLS_WANT_POLLOUT: > + return WANT_POLLOUT; Looks like http_close() failure is currently silent, otherwise the missing success case would have been noticed: case 0: return 0; > + case -1: > + warnx("%s: TLS close: %s", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > + } > + } > + > + return -1; > +} > + > +static int > +http_parse_uri(char *uri, char **ohost, char **oport, char **opath) > +{ > + char *host, *port = NULL, *path; > + char *hosttail; > + > + if (strncasecmp(uri, "https://", 8) != 0) { > + warnx("%s: not using https schema", http_info(uri)); > + return -1; > + } > + host = uri + 8; > + if ((path = strchr(host, '/')) == NULL) { > + warnx("%s: missing https path", http_info(uri)); > + return -1; > + } > + if (path - uri > INT_MAX - 1) { > + warnx("%s: preposterous host length", http_info(uri)); > + return -1; > + } > + if (*host == '[') { > + char *scope; > + if ((hosttail = memrchr(host, ']', path - host)) != NULL && > + (hosttail[1] == '/' || hosttail[1] == ':')) > + host++; > + if (hosttail[1] == ':') > + port = hosttail + 1; > + if ((scope = memchr(host, '%', hosttail - host)) != NULL) > + hosttail = scope; > + } else { > + if ((hosttail = memrchr(host, ':', path - host)) != NULL) > + port = hosttail + 1; > + else > + hosttail = path; > + } > + > + if ((host = strndup(host, hosttail - host)) == NULL) > + err(1, "strndup"); > + if (port) { > + if ((port = strndup(port, path - port)) == NULL) > + err(1, "strndup"); > + } else { > + if ((port = strdup("443")) == NULL) > + err(1, "strdup"); > + } > + /* do not include the initial / in path */ > + path++; > + > + *ohost = host; > + *oport = port; > + *opath = path; > + > + return 0; > +} > + > + > +static struct http_connection * > +http_new(size_t id, char *uri, char *modified_since, int outfd) > +{ > + struct http_connection *conn; > + char *host, *port, *path; > + > + if (http_parse_uri(uri, &host, &port, &path) == -1) { > + free(uri); > + free(modified_since); > + return NULL; > + } > + > + if ((conn = calloc(1, sizeof(*conn))) == NULL) > + err(1, NULL); > + > + conn->id = id; > + conn->fd = -1; > + conn->outfd = outfd; > + conn->host = host; > + conn->port = port; > + conn->path = path; > + conn->url = uri; > + conn->modified_since = modified_since; > + conn->state = STATE_INIT; > + > + /* TODO proxy support (overload of host and port) */ > + > + if (http_resolv(conn, host, port) == -1) { > + http_free(conn); > + return NULL; > + } > + > + return conn; > +} > + > +static int > +http_redirect(struct http_connection *conn, char *uri) > +{ > + char *host, *port, *path; > + > + warnx("redirect to %s", http_info(uri)); > + > + if (http_parse_uri(uri, &host, &port, &path) == -1) { > + free(uri); > + return -1; > + } > + > + free(conn->host); > + conn->host = host; > + free(conn->port); > + conn->port = port; > + free(conn->url); > + conn->url = uri; move handling of url to the beginning > + conn->path = path; conn->modified_since omitted? > + free(conn->last_modified); > + conn->last_modified = NULL; > + free(conn->buf); > + conn->buf = NULL; > + conn->bufpos = 0; > + conn->bufsz = 0; > + tls_close(conn->tls); > + tls_free(conn->tls); > + conn->tls = NULL; > + close(conn->fd); > + conn->state = STATE_INIT; > + > + /* TODO proxy support (overload of host and port) */ > + > + if (http_resolv(conn, host, port) == -1) > + return -1; > + > + return 0; > +} > + > +static int > +http_connect(struct http_connection *conn) > +{ > + char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST]; > + char *cause = "unknown"; > + > + if (conn->fd != -1) { > + close(conn->fd); > + conn->fd = -1; > + } > + > + /* start the loop below with first or next address */ > + if (conn->res == NULL) > + conn->res = conn->res0; > + else > + conn->res = conn->res->ai_next; > + for (; conn->res != NULL; conn->res = conn->res->ai_next) { > + struct addrinfo *res = conn->res; > + int fd, error, save_errno; > + > + if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, > + sizeof(hbuf), pbuf, sizeof(pbuf), > + NI_NUMERICHOST | NI_NUMERICSERV) != 0) > + strlcpy(hbuf, "(unknown)", sizeof(hbuf)); > + > + fd = socket(res->ai_family, > + res->ai_socktype | SOCK_NONBLOCK, res->ai_protocol); > + if (fd == -1) { > + cause = "socket"; > + continue; > + } > + conn->fd = fd; > + > + if (http_bindaddr.ss_family == res->ai_family) { > + if (bind(conn->fd, (struct sockaddr *)&http_bindaddr, > + res->ai_addrlen) == -1) > + warn("%s: bind", http_info(conn->url)); > + } > + > + error = connect(conn->fd, res->ai_addr, res->ai_addrlen); > + if (error == -1) { > + if (errno == EINPROGRESS) { > + /* waiting for connect to finish. */ > + return WANT_POLLOUT; > + } else { > + save_errno = errno; > + close(conn->fd); > + conn->fd = -1; > + errno = save_errno; > + cause = "connect"; > + continue; > + } > + } > + > + /* TODO proxy connect */ > +#if 0 > + if (proxyenv) > + proxy_connect(conn->fd, sslhost, proxy_credentials); */ > +#endif > + } > + freeaddrinfo(conn->res0); > + conn->res0 = NULL; > + if (conn->fd == -1) { > + warn("%s: %s", http_info(conn->url), cause); > + return -1; > + } > + return 0; > +} > + > +static int > +http_finish_connect(struct http_connection *conn) > +{ > + int error = 0; > + socklen_t len; > + > + len = sizeof(error); > + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &error, &len) == -1) { > + warn("%s: getsockopt SO_ERROR", http_info(conn->url)); > + /* connection will be closed by http_connect() */ > + return -1; > + } > + if (error != 0) { > + errno = error; > + warn("%s: connect", http_info(conn->url)); > + return -1; > + } > + > + return 0; > +} > + > +static int > +http_tls_handshake(struct http_connection *conn) > +{ > + switch (tls_handshake(conn->tls)) { > + case 0: > + return 0; > + case TLS_WANT_POLLIN: > + return WANT_POLLIN; > + case TLS_WANT_POLLOUT: > + return WANT_POLLOUT; > + } > + warnx("%s: TLS handshake: %s", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > +} > + > +static int > +http_tls_connect(struct http_connection *conn) > +{ > + if ((conn->tls = tls_client()) == NULL) { > + warn("tls_client"); > + return -1; > + } > + if (tls_configure(conn->tls, tls_config) != 0) { > + warnx("%s: TLS configuration: %s\n", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > + } > + if (tls_connect_socket(conn->tls, conn->fd, conn->host) != 0) { > + warnx("%s: TLS connect: %s\n", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > + } > + return http_tls_handshake(conn); > +} > + > +static int > +http_request(struct http_connection *conn) > +{ > + char *host, *epath, *modified_since; > + int with_port = 0; > + > + /* TODO adjust request for HTTP proxy setups */ > + > + /* > + * Send port number only if it's specified and does not equal > + * the default. Some broken HTTP servers get confused if you explicitly > + * send them the port number. > + */ > + if (conn->port && strcmp(conn->port, "443") != 0) > + with_port = 1; > + > + /* Construct the Host header from host and port info */ > + if (strchr(conn->host, ':')) { > + if (asprintf(&host, "[%s]%s%s", conn->host, > + with_port ? ":" : "", with_port ? conn->port : "") == -1) > + err(1, NULL); > + > + } else { > + if (asprintf(&host, "%s%s%s", conn->host, > + with_port ? ":" : "", with_port ? conn->port : "") == -1) > + err(1, NULL); > + } > + > + /* > + * Construct and send the request. Proxy requests don't want leading /. > + */ > + epath = url_encode(conn->path); > + > + modified_since = NULL; > + if (conn->modified_since) { > + if (asprintf(&modified_since, "If-Modified-Since: %s\r\n", > + conn->modified_since) == -1) > + err(1, NULL); > + } > + > + free(conn->buf); > + conn->bufpos = 0; > + if ((conn->bufsz = asprintf(&conn->buf, > + "GET /%s HTTP/1.1\r\n" > + "Connection: close\r\n" > + "User-Agent: " HTTP_USER_AGENT "\r\n" > + "Host: %s\r\n%s\r\n", > + epath, host, > + modified_since ? modified_since : "")) == -1) > + err(1, NULL); > + > + free(epath); > + free(host); > + free(modified_since); > + > + return WANT_POLLOUT; > +} > + > +static int > +http_write(struct http_connection *conn) > +{ > + ssize_t s; > + > + s = tls_write(conn->tls, conn->buf + conn->bufpos, > + conn->bufsz - conn->bufpos); > + if (s == -1) { > + warnx("%s: TLS write: %s", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > + } else if (s == TLS_WANT_POLLIN) { > + return WANT_POLLIN; > + } else if (s == TLS_WANT_POLLOUT) { > + return WANT_POLLOUT; > + } > + > + conn->bufpos += s; > + if (conn->bufpos == conn->bufsz) > + return 0; > + return WANT_POLLOUT; > +} > + > +static int > +http_parse_status(struct http_connection *conn, char *buf) > +{ > + const char *errstr; > + char *cp, ststr[4]; > + char gerror[200]; > + int status; > + > + cp = strchr(buf, ' '); > + if (cp == NULL) { > + warnx("Improper response from %s", http_info(conn->url)); > + return -1; > + } else > + cp++; > + > + strlcpy(ststr, cp, sizeof(ststr)); > + status = strtonum(ststr, 200, 599, &errstr); > + if (errstr) { > + strnvis(gerror, cp, sizeof gerror, VIS_SAFE); > + warnx("Error retrieving %s: %s", http_info(conn->url), gerror); > + return -1; > + } > + > + switch (status) { > + case 301: > + case 302: > + case 303: > + case 307: > + if (conn->redirect_loop++ > 10) { > + warnx("%s: Too many redirections requested", > + http_info(conn->url)); > + return -1; > + } > + /* FALLTHROUGH */ > + case 200: > + case 206: > + case 304: > + conn->status = status; > + break; > + default: > + strnvis(gerror, cp, sizeof gerror, VIS_SAFE); > + warnx("Error retrieving %s: %s", http_info(conn->url), gerror); > + break; > + } > + > + return 0; > +} > + > +static inline int > +http_isredirect(struct http_connection *conn) > +{ > + if ((conn->status >= 301 && conn->status <= 303) || > + conn->status == 307) > + return 1; > + return 0; > +} > + > +static int > +http_parse_header(struct http_connection *conn, char *buf) > +{ > +#define CONTENTLEN "Content-Length: " > +#define LOCATION "Location: " > +#define TRANSFER_ENCODING "Transfer-Encoding: " > +#define LAST_MODIFIED "Last-Modified: " > + const char *errstr; > + char *cp, *redirurl; > + char *locbase, *loctail; > + > + cp = buf; > + /* empty line, end of header */ > + if (*cp == '\0') > + return 0; > + else if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) { > + size_t s; > + cp += sizeof(CONTENTLEN) - 1; > + if ((s = strcspn(cp, " \t"))) > + *(cp+s) = 0; > + conn->filesize = strtonum(cp, 0, LLONG_MAX, &errstr); > + if (errstr != NULL) { > + warnx("Improper response from %s", > + http_info(conn->url)); > + return -1; > + } > + } else if (http_isredirect(conn) && > + strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) { > + cp += sizeof(LOCATION) - 1; > + /* > + * If there is a colon before the first slash, this URI > + * is not relative. RFC 3986 4.2 > + */ > + if (cp[strcspn(cp, ":/")] != ':') { > + /* XXX doesn't handle protocol-relative URIs */ > + if (*cp == '/') { > + locbase = NULL; > + cp++; > + } else { > + locbase = strdup(conn->path); > + if (locbase == NULL) > + err(1, ""); > + loctail = strchr(locbase, '#'); > + if (loctail != NULL) > + *loctail = '\0'; > + loctail = strchr(locbase, '?'); > + if (loctail != NULL) > + *loctail = '\0'; > + loctail = strrchr(locbase, '/'); > + if (loctail == NULL) { > + free(locbase); > + locbase = NULL; > + } else > + loctail[1] = '\0'; > + } > + /* Contruct URL from relative redirect */ Typo: Construct > + if (asprintf(&redirurl, "%.*s/%s%s", > + (int)(conn->path - conn->url), conn->url, > + locbase ? locbase : "", > + cp) == -1) > + err(1, "Cannot build redirect URL"); > + free(locbase); > + } else if ((redirurl = strdup(cp)) == NULL) > + err(1, "Cannot build redirect URL"); > + loctail = strchr(redirurl, '#'); > + if (loctail != NULL) > + *loctail = '\0'; > + return http_redirect(conn, redirurl); > + } else if (strncasecmp(cp, TRANSFER_ENCODING, > + sizeof(TRANSFER_ENCODING) - 1) == 0) { > + cp += sizeof(TRANSFER_ENCODING) - 1; > + cp[strcspn(cp, " \t")] = '\0'; > + if (strcasecmp(cp, "chunked") == 0) > + conn->chunked = 1; > + } else if (strncasecmp(cp, LAST_MODIFIED, > + sizeof(LAST_MODIFIED) - 1) == 0) { > + cp += sizeof(LAST_MODIFIED) - 1; > + conn->last_modified = strdup(cp); > + } > + > + return 1; > +} > + > +static char * > +http_get_line(struct http_connection *conn) > +{ > + char *end, *line; > + size_t len; > + > + end = memchr(conn->buf, '\n', conn->bufpos); > + if (end == NULL) > + return NULL; > + > + len = end - conn->buf; > + while (len > 0 && conn->buf[len - 1] == '\r') > + --len; > + line = strndup(conn->buf, len); Should strndup failure not be a fatal error? > + > + /* consume line including \n */ > + end++; > + conn->bufpos -= end - conn->buf; > + memmove(conn->buf, end, conn->bufpos); > + > + return line; > +} > + > +static int > +http_parse_chunked(struct http_connection *conn, char *buf) > +{ > + char *header = buf; > + char *end; > + int chunksize; > + > + /* ignore empty lines, used between chunk and next header */ > + if (*header == '\0') > + return 1; > + > + /* strip CRLF and any optional chunk extension */ > + header[strcspn(header, ";\r\n")] = '\0'; > + errno = 0; > + chunksize = strtoul(header, &end, 16); > + if (errno || header[0] == '\0' || *end != '\0' || > + chunksize > INT_MAX) { > + warnx("%s: Invalid chunk size", http_info(conn->url)); > + return -1; > + } > + conn->chunksz = chunksize; > + > + if (conn->chunksz == 0) { > + http_done(conn, 1); > + return 0; > + } > + > + return 1; > +} > + > +static int > +http_read(struct http_connection *conn) > +{ > + ssize_t s; > + char *buf; > + > + s = tls_read(conn->tls, conn->buf + conn->bufpos, > + conn->bufsz - conn->bufpos); > + if (s == -1) { > + warn("%s: TLS read: %s", http_info(conn->url), > + tls_error(conn->tls)); > + return -1; > + } else if (s == TLS_WANT_POLLIN) { > + return WANT_POLLIN; > + } else if (s == TLS_WANT_POLLOUT) { > + return WANT_POLLOUT; > + } > + > + if (s == 0 && conn->bufpos == 0) { > + warnx("%s: short read, connection closed", > + http_info(conn->url)); > + return -1; > + } > + > + conn->bufpos += s; > + > + switch (conn->state) { > + case STATE_RESPONSE_STATUS: > + buf = http_get_line(conn); > + if (buf == NULL) > + return WANT_POLLIN; > + if (http_parse_status(conn, buf) == -1) { > + free(buf); > + return -1; > + } > + free(buf); > + conn->state = STATE_RESPONSE_HEADER; > + /* FALLTHROUGH */ > + case STATE_RESPONSE_HEADER: > + while ((buf = http_get_line(conn)) != NULL) { > + int rv; > + > + rv = http_parse_header(conn, buf); > + free(buf); > + if (rv != 1) > + return rv; > + } > + return WANT_POLLIN; > + case STATE_RESPONSE_DATA: > + if (conn->bufpos == conn->bufsz || > + conn->filesize - conn->bytes <= (off_t)conn->bufpos) > + return 0; > + return WANT_POLLIN; > + case STATE_RESPONSE_CHUNKED: > + while (conn->chunksz == 0) { > + buf = http_get_line(conn); > + if (buf == NULL) > + return WANT_POLLIN; > + switch (http_parse_chunked(conn, buf)) { > + case -1: > + free(buf); > + return -1; > + case 0: > + free(buf); > + return 0; > + } > + free(buf); > + } > + > + if (conn->bufpos == conn->bufsz || > + conn->chunksz <= conn->bufpos) > + return 0; > + return WANT_POLLIN; > + default: > + errx(1, "unexpected http state"); > + } > +} > + > +static int > +data_write(struct http_connection *conn) > +{ > + ssize_t s; > + size_t bsz = conn->bufpos; > + > + if (conn->filesize - conn->bytes < (off_t)bsz) > + bsz = conn->filesize - conn->bytes; > + > + s = write(conn->outfd, conn->buf, bsz); > + if (s == -1) { > + warn("%s: Data write", http_info(conn->url)); > + return -1; > + } > + > + conn->bufpos -= s; > + conn->bytes += s; > + memmove(conn->buf, conn->buf + s, conn->bufpos); > + > + if (conn->bytes == conn->filesize) { > + http_done(conn, 1); > + return 0; > + } > + > + if (conn->bufpos == 0) > + return 0; > + > + return WANT_POLLOUT; > +} > + > +static int > +chunk_write(struct http_connection *conn) > +{ > + ssize_t s; > + size_t bsz = conn->bufpos; > + > + if (bsz > conn->chunksz) > + bsz = conn->chunksz; > + > + s = write(conn->outfd, conn->buf, bsz); > + if (s == -1) { > + warn("%s: Chunk write", http_info(conn->url)); > + return -1; > + } > + > + conn->bufpos -= s; > + conn->chunksz -= s; > + conn->bytes += s; > + memmove(conn->buf, conn->buf + s, conn->bufpos); > + > + if (conn->bufpos == 0 || conn->chunksz == 0) > + return 0; > + > + return WANT_POLLOUT; > +} > + > +static int > +http_handle(struct http_connection *conn) > +{ > + switch (conn->state) { > + case STATE_INIT: > + return 0; > + case STATE_CONNECT: > + if (http_finish_connect(conn) == -1) > + /* something went wrong, try other host */ > + return http_connect(conn); > + return 0; > + case STATE_TLSCONNECT: > + return http_tls_handshake(conn); > + case STATE_REQUEST: > + return http_write(conn); > + case STATE_RESPONSE_STATUS: > + case STATE_RESPONSE_HEADER: > + case STATE_RESPONSE_DATA: > + case STATE_RESPONSE_CHUNKED: > + return http_read(conn); > + case STATE_WRITE_DATA: > + if (conn->chunked) > + return chunk_write(conn); > + else > + return data_write(conn); > + case STATE_DONE: > + return http_close(conn); > + case STATE_FREE: > + errx(1, "bad http state"); > + } > +} > + > +static int > +http_nextstep(struct http_connection *conn) > +{ > + switch (conn->state) { > + case STATE_INIT: > + conn->state = STATE_CONNECT; > + return http_connect(conn); > + case STATE_CONNECT: > + conn->state = STATE_TLSCONNECT; > + return http_tls_connect(conn); > + case STATE_TLSCONNECT: > + conn->state = STATE_REQUEST; > + return http_request(conn); > + case STATE_REQUEST: > + conn->state = STATE_RESPONSE_STATUS; > + free(conn->buf); > + /* allocate the read buffer */ > + if ((conn->buf = malloc(HTTP_BUF_SIZE)) == NULL) > + err(1, NULL); > + conn->bufpos = 0; > + conn->bufsz = HTTP_BUF_SIZE; > + return WANT_POLLIN; > + case STATE_RESPONSE_STATUS: > + conn->state = STATE_RESPONSE_HEADER; > + return WANT_POLLIN; > + case STATE_RESPONSE_HEADER: > + if (conn->status == 200) > + conn->state = STATE_RESPONSE_DATA; > + else { > + http_done(conn, 0); > + return http_close(conn); > + } > + return WANT_POLLIN; > + case STATE_RESPONSE_DATA: > + conn->state = STATE_WRITE_DATA; > + return WANT_POLLOUT; > + case STATE_RESPONSE_CHUNKED: > + conn->state = STATE_WRITE_DATA; > + return WANT_POLLOUT; > + case STATE_WRITE_DATA: > + if (conn->chunked) > + conn->state = STATE_RESPONSE_CHUNKED; > + else > + conn->state = STATE_RESPONSE_DATA; > + return WANT_POLLIN; > + case STATE_DONE: > + return http_close(conn); > + case STATE_FREE: > + errx(1, "bad http state"); > + } > +} > + > +static int > +http_do(struct http_connection *conn) > +{ > + switch (http_handle(conn)) { > + case -1: > + /* connection failure */ > + http_free(conn); > + return -1; > + case 0: > + switch (http_nextstep(conn)) { > + case WANT_POLLIN: > + conn->events = POLLIN; > + break; > + case WANT_POLLOUT: > + conn->events = POLLOUT; > + break; > + case -1: > + http_free(conn); > + return -1; > + } > + break; > + case WANT_POLLIN: > + conn->events = POLLIN; > + break; > + case WANT_POLLOUT: > + conn->events = POLLOUT; > + break; > + } > + return 0; > +} > + > +void > +proc_http(char *bind_addr, int fd) > +{ > + struct http_connection *http_conns[MAX_CONNECTIONS]; > + struct pollfd pfds[MAX_CONNECTIONS + 1]; > + size_t i; > + int active_connections; > + > + if (bind_addr) { > + struct addrinfo hints, *res; > + > + bzero(&hints, sizeof(hints)); > + hints.ai_family = AF_UNSPEC; > + hints.ai_socktype = SOCK_DGRAM; /*dummy*/ > + hints.ai_flags = AI_NUMERICHOST; > + if (getaddrinfo(bind_addr, NULL, &hints, &res) == 0) { > + memcpy(&http_bindaddr, res->ai_addr, res->ai_addrlen); > + freeaddrinfo(res); > + } > + } > + http_setup(); > + > + if (pledge("stdio inet dns recvfd", NULL) == -1) > + err(1, "pledge"); > + > + memset(&http_conns, 0, sizeof(http_conns)); > + memset(&pfds, 0, sizeof(pfds)); > + pfds[MAX_CONNECTIONS].fd = fd; > + > + msgbuf_init(&msgq); > + msgq.fd = fd; > + > + for (;;) { > + active_connections = 0; > + for (i = 0; i < MAX_CONNECTIONS; i++) { > + struct http_connection *conn = http_conns[i]; > + if (conn == NULL) { > + pfds[i].fd = -1; > + continue; > + } > + if (conn->state == STATE_WRITE_DATA) > + pfds[i].fd = conn->outfd; > + else > + pfds[i].fd = conn->fd; > + > + pfds[i].events = conn->events; > + active_connections++; > + } > + pfds[MAX_CONNECTIONS].events = 0; > + if (active_connections < MAX_CONNECTIONS) > + pfds[MAX_CONNECTIONS].events |= POLLIN; > + if (msgq.queued) > + pfds[MAX_CONNECTIONS].events |= POLLOUT; > + > + if (poll(pfds, sizeof(pfds) / sizeof(pfds[0]), INFTIM) == -1) > + err(1, "poll"); > + > + if (pfds[MAX_CONNECTIONS].revents & POLLHUP) > + break; > + > + if (pfds[MAX_CONNECTIONS].revents & POLLIN) { > + struct http_connection *h; > + size_t id; > + int outfd; > + char *uri; > + char *mod; > + > + outfd = io_recvfd(fd, &id, sizeof(id)); > + io_str_read(fd, &uri); > + io_str_read(fd, &mod); > + > + h = http_new(id, uri, mod, outfd); > + if (h == NULL) { > + close(outfd); > + http_fail(id); > + } else > + for (i = 0; i < MAX_CONNECTIONS; i++) { > + if (http_conns[i] != NULL) > + continue; > + http_conns[i] = h; > + if (http_do(h) == -1) > + http_conns[i] = NULL; > + break; > + } > + } > + if (pfds[MAX_CONNECTIONS].revents & POLLOUT) { > + switch (msgbuf_write(&msgq)) { > + case 0: > + errx(1, "write: connection closed"); > + case -1: > + err(1, "write"); > + } > + } > + for (i = 0; i < MAX_CONNECTIONS; i++) { > + struct http_connection *conn = http_conns[i]; > + > + if (conn == NULL) > + continue; > + /* event not ready */ > + if (!(pfds[i].revents & (conn->events | POLLHUP))) > + continue; > + > + if (http_do(conn) == -1) > + http_conns[i] = NULL; > + } > + } > +} > Index: io.c > =================================================================== > RCS file: /cvs/src/usr.sbin/rpki-client/io.c,v > retrieving revision 1.12 > diff -u -p -r1.12 io.c > --- io.c 8 Jan 2021 08:09:07 -0000 1.12 > +++ io.c 18 Feb 2021 09:54:16 -0000 > @@ -16,9 +16,11 @@ > */ > > #include <sys/queue.h> > +#include <sys/socket.h> > > #include <assert.h> > #include <err.h> > +#include <errno.h> > #include <fcntl.h> > #include <stdint.h> > #include <stdlib.h> > @@ -145,4 +147,79 @@ io_str_read(int fd, char **res) > if ((*res = calloc(sz + 1, 1)) == NULL) > err(1, NULL); > io_simple_read(fd, *res, sz); > +} > + > +/* > + * Read data from socket but receive a file descriptor at the same time. > + */ > +int > +io_recvfd(int fd, void *res, size_t sz) > +{ > + struct iovec iov; > + struct msghdr msg; > + struct cmsghdr *cmsg; > + union { > + struct cmsghdr hdr; > + char buf[CMSG_SPACE(sizeof(int))]; > + } cmsgbuf; > + int outfd = -1; > + char *b = res; > + ssize_t n; > + > + memset(&msg, 0, sizeof(msg)); > + memset(&cmsgbuf, 0, sizeof(cmsgbuf)); > + > + iov.iov_base = res; > + iov.iov_len = sz; > + > + msg.msg_iov = &iov; > + msg.msg_iovlen = 1; > + msg.msg_control = &cmsgbuf.buf; > + msg.msg_controllen = sizeof(cmsgbuf.buf); > + > + while ((n = recvmsg(fd, &msg, 0)) == -1) { > + if (errno == EINTR) > + continue; > + err(1, "recvmsg"); > + } > + > + if (n == 0) > + errx(1, "recvmsg: unexpected end of file"); > + > + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; > + cmsg = CMSG_NXTHDR(&msg, cmsg)) { > + if (cmsg->cmsg_level == SOL_SOCKET && > + cmsg->cmsg_type == SCM_RIGHTS) { > + int i, j, f; > + > + j = ((char *)cmsg + cmsg->cmsg_len - > + (char *)CMSG_DATA(cmsg)) / sizeof(int); > + for (i = 0; i < j; i++) { > + f = ((int *)CMSG_DATA(cmsg))[i]; > + if (i == 0) > + outfd = f; > + else > + close(f); > + } > + } > + } > + > + b += n; > + sz -= n; > + while (sz > 0) { > + /* short receive */ > + n = recv(fd, b, sz, 0); > + if (n == -1) { > + if (errno == EINTR) > + continue; > + err(1, "recv"); > + } > + if (n == 0) > + errx(1, "recv: unexpected end of file"); > + > + b += n; > + sz -= n; > + } > + > + return outfd; > } > Index: main.c > =================================================================== > RCS file: /cvs/src/usr.sbin/rpki-client/main.c,v > retrieving revision 1.105 > diff -u -p -r1.105 main.c > --- main.c 23 Feb 2021 14:25:29 -0000 1.105 > +++ main.c 25 Feb 2021 09:34:00 -0000 > @@ -51,10 +51,12 @@ > /* > * An rsync repository. > */ > +#define REPO_MAX_URI 2 > struct repo { > char *repouri; /* CA repository base URI */ > char *local; /* local path name */ > - char *uris[2]; /* URIs to fetch from */ > + char *temp; /* temporary file / dir */ > + char *uris[REPO_MAX_URI]; /* URIs to fetch from */ > size_t id; /* identifier (array index) */ > int uriidx; /* which URI is fetched */ > int loaded; /* whether loaded or not */ > @@ -91,7 +93,7 @@ RB_HEAD(filepath_tree, filepath); > RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); > > static struct filepath_tree fpt = RB_INITIALIZER(&fpt); > -static struct msgbuf procq, rsyncq; > +static struct msgbuf procq, rsyncq, httpq; > static int cachefd; > > const char *bird_tablename = "ROAS"; > @@ -101,6 +103,9 @@ int noop; > > struct stats stats; > > +static void repo_fetch(struct repo *); > +static char *ta_filename(const struct repo *, int); > + > /* > * Log a message to stderr if and only if "verbose" is non-zero. > * This uses the err(3) functionality. > @@ -279,6 +284,70 @@ repo_alloc(void) > } > > static void > +http_ta_fetch(struct repo *rp) > +{ > + struct ibuf *b; > + int filefd; > + > + rp->temp = ta_filename(rp, 1); > + > + filefd = mkostempat(cachefd, rp->temp, O_CLOEXEC); > + if (filefd == -1) { > + err(1, "mkostempat: %s", rp->temp); > + /* XXX switch to soft fail and restart with next file */ > + } > + if (fchmod(filefd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) > + warn("fchmod: %s", rp->temp); > + > + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) > + err(1, NULL); > + io_simple_buffer(b, &rp->id, sizeof(rp->id)); > + io_str_buffer(b, rp->uris[rp->uriidx]); > + /* TODO last modified time */ > + io_str_buffer(b, NULL); > + /* pass file as fd */ > + b->fd = filefd; > + ibuf_close(&httpq, b); > +} > + > +static int > +http_done(struct repo *rp, int ok) > +{ > + if (rp->repouri == NULL) { > + /* Move downloaded TA file into place, or unlink on failure. */ > + if (ok) { > + char *file; > + > + file = ta_filename(rp, 0); > + if (renameat(cachefd, rp->temp, cachefd, file) == -1) > + warn("rename to %s", file); > + } else { > + if (unlinkat(cachefd, rp->temp, 0) == -1) > + warn("unlink %s", rp->temp); > + } > + free(rp->temp); > + rp->temp = NULL; > + } > + > + if (!ok && rp->uriidx < REPO_MAX_URI - 1 && > + rp->uris[rp->uriidx + 1] != NULL) { > + logx("%s: load from network failed, retry", rp->local); > + > + rp->uriidx++; > + repo_fetch(rp); > + return 0; > + } > + > + if (ok) > + logx("%s: loaded from network", rp->local); > + else > + logx("%s: load from network failed, " > + "fallback to cache", rp->local); > + > + return 1; > +} > + > +static void > repo_fetch(struct repo *rp) > { > struct ibuf *b; > @@ -293,20 +362,31 @@ repo_fetch(struct repo *rp) > > /* > * Create destination location. > - * Build up the tree to this point because GPL rsync(1) > - * will not build the destination for us. > + * Build up the tree to this point. > */ > > if (mkpathat(cachefd, rp->local) == -1) > err(1, "%s", rp->local); > > - logx("%s: pulling from network", rp->local); > - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) > - err(1, NULL); > - io_simple_buffer(b, &rp->id, sizeof(rp->id)); > - io_str_buffer(b, rp->local); > - io_str_buffer(b, rp->uris[0]); > - ibuf_close(&rsyncq, b); > + logx("%s: pulling from %s", rp->local, rp->uris[rp->uriidx]); > + > + if (strncasecmp(rp->uris[rp->uriidx], "rsync://", 8) == 0) { > + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) > + err(1, NULL); > + io_simple_buffer(b, &rp->id, sizeof(rp->id)); > + io_str_buffer(b, rp->local); > + io_str_buffer(b, rp->uris[rp->uriidx]); > + ibuf_close(&rsyncq, b); > + } else { > + /* > + * Two cases for https. TA files load directly while > + * for RRDP XML files are downloaded an parsed to build > + * the repo. TA repos have a NULL repouri. > + */ > + if (rp->repouri == NULL) { > + http_ta_fetch(rp); > + } > + } > } > > /* > @@ -333,13 +413,11 @@ ta_lookup(const struct tal *tal) > rp = repo_alloc(); > rp->local = local; > for (i = 0, j = 0; i < tal->urisz && j < 2; i++) { > - if (strncasecmp(tal->uri[i], "rsync://", 8) != 0) > - continue; /* ignore non rsync URI for now */ > if ((rp->uris[j++] = strdup(tal->uri[i])) == NULL) > err(1, "strdup"); > } > if (j == 0) > - errx(1, "TAL file has no rsync:// URI"); > + errx(1, "TAL %s has no URI", tal->descr); > > repo_fetch(rp); > return rp; > @@ -379,6 +457,23 @@ repo_lookup(const char *uri) > return rp; > } > > +static char * > +ta_filename(const struct repo *repo, int temp) > +{ > + const char *file; > + char *nfile; > + > + /* does not matter which URI, all end with same filename */ > + file = strrchr(repo->uris[0], '/'); > + assert(file); > + > + if (asprintf(&nfile, "%s%s%s", repo->local, file, > + temp ? ".XXXXXXXX": "") == -1) > + err(1, "asprintf"); > + > + return nfile; > +} > + > /* > * Build local file name base on the URI and the repo info. > */ > @@ -521,18 +616,13 @@ queue_add_from_tal(struct entityq *q, co > { > char *nfile; > const struct repo *repo; > - const char *uri; > > assert(tal->urisz); > > /* Look up the repository. */ > repo = ta_lookup(tal); > > - uri = strrchr(repo->uris[0], '/'); > - assert(uri); > - if (asprintf(&nfile, "%s/%s", repo->local, uri + 1) == -1) > - err(1, "asprintf"); > - > + nfile = ta_filename(repo, 0); > entityq_add(q, nfile, RTYPE_CER, repo, tal->pkey, > tal->pkeysz, tal->descr); > } > @@ -771,13 +861,13 @@ suicide(int sig __attribute__((unused))) > int > main(int argc, char *argv[]) > { > - int rc = 1, c, proc, st, rsync, > + int rc = 1, c, st, proc, rsync, http, > fl = SOCK_STREAM | SOCK_CLOEXEC; > size_t i, outsz = 0, talsz = 0; > - pid_t procpid, rsyncpid; > + pid_t procpid, rsyncpid, httppid; > int fd[2]; > struct entityq q; > - struct pollfd pfd[2]; > + struct pollfd pfd[3]; > struct roa **out = NULL; > char *rsync_prog = "openrsync"; > char *bind_addr = NULL; > @@ -805,7 +895,8 @@ main(int argc, char *argv[]) > cachedir = RPKI_PATH_BASE_DIR; > outputdir = RPKI_PATH_OUT_DIR; > > - if (pledge("stdio rpath wpath cpath fattr proc exec unveil", NULL) == > -1) > + if (pledge("stdio rpath wpath cpath inet fattr dns sendfd recvfd " > + "proc exec unveil", NULL) == -1) > err(1, "pledge"); > > while ((c = getopt(argc, argv, "b:Bcd:e:jnos:t:T:v")) != -1) > @@ -953,15 +1044,48 @@ main(int argc, char *argv[]) > } else > rsync = -1; > > - assert(rsync != proc); > + /* > + * Create a process that will fetch data via https. > + * With every request the http process receives a file descriptor > + * where the data should be written to. > + */ > + > + if (!noop) { > + if (socketpair(AF_UNIX, fl, 0, fd) == -1) > + err(1, "socketpair"); > + if ((httppid = fork()) == -1) > + err(1, "fork"); > + > + if (httppid == 0) { > + close(proc); > + close(rsync); > + close(fd[1]); > + > + /* change working directory to the cache directory */ > + if (fchdir(cachefd) == -1) > + err(1, "fchdir"); > + > + if (pledge("stdio rpath inet dns recvfd", NULL) == -1) > + err(1, "pledge"); > + > + proc_http(bind_addr, fd[0]); > + /* NOTREACHED */ > + } > + > + close(fd[0]); > + http = fd[1]; > + } else > + http = -1; > > - if (pledge("stdio rpath wpath cpath fattr", NULL) == -1) > + if (pledge("stdio rpath wpath cpath fattr sendfd", NULL) == -1) > err(1, "pledge"); > > msgbuf_init(&procq); > msgbuf_init(&rsyncq); > + msgbuf_init(&httpq); > procq.fd = proc; > rsyncq.fd = rsync; > + httpq.fd = http; > > /* > * The main process drives the top-down scan to leaf ROAs using > @@ -971,6 +1095,7 @@ main(int argc, char *argv[]) > > pfd[0].fd = rsync; > pfd[1].fd = proc; > + pfd[2].fd = http; > > /* > * Prime the process with our TAL file. > @@ -984,22 +1109,27 @@ main(int argc, char *argv[]) > while (entity_queue > 0 && !killme) { > pfd[0].events = POLLIN; > if (rsyncq.queued) > - pfd[0].events = POLLOUT; > + pfd[0].events |= POLLOUT; > pfd[1].events = POLLIN; > if (procq.queued) > - pfd[1].events = POLLOUT; > + pfd[1].events |= POLLOUT; > + pfd[2].events = POLLIN; > + if (httpq.queued) > + pfd[2].events |= POLLOUT; > > - if ((c = poll(pfd, 2, INFTIM)) == -1) { > + if ((c = poll(pfd, 3, INFTIM)) == -1) { > if (errno == EINTR) > continue; > err(1, "poll"); > } > > if ((pfd[0].revents & (POLLERR|POLLNVAL)) || > - (pfd[1].revents & (POLLERR|POLLNVAL))) > + (pfd[1].revents & (POLLERR|POLLNVAL)) || > + (pfd[2].revents & (POLLERR|POLLNVAL))) > errx(1, "poll: bad fd"); > if ((pfd[0].revents & POLLHUP) || > - (pfd[1].revents & POLLHUP)) > + (pfd[1].revents & POLLHUP) || > + (pfd[2].revents & POLLHUP)) > errx(1, "poll: hangup"); > > if (pfd[0].revents & POLLOUT) { > @@ -1018,9 +1148,18 @@ main(int argc, char *argv[]) > err(1, "write"); > } > } > + if (pfd[2].revents & POLLOUT) { > + switch (msgbuf_write(&httpq)) { > + case 0: > + errx(1, "write: connection closed"); > + case -1: > + err(1, "write"); > + } > + } > + > > /* > - * Check the rsync process. > + * Check the rsync and http process. > * This means that one of our modules has completed > * downloading and we can flush the module requests into > * the parser process. > @@ -1033,17 +1172,31 @@ main(int argc, char *argv[]) > assert(i < rt.reposz); > > assert(!rt.repos[i].loaded); > - rt.repos[i].loaded = 1; > if (ok) > logx("%s: loaded from network", > rt.repos[i].local); > else > logx("%s: load from network failed, " > "fallback to cache", rt.repos[i].local); > + rt.repos[i].loaded = 1; > stats.repos++; > entityq_flush(&q, &rt.repos[i]); > } > > + if ((pfd[2].revents & POLLIN)) { > + int ok; > + io_simple_read(http, &i, sizeof(size_t)); > + io_simple_read(http, &ok, sizeof(ok)); > + assert(i < rt.reposz); > + > + assert(!rt.repos[i].loaded); > + if (http_done(&rt.repos[i], ok)) { > + rt.repos[i].loaded = 1; > + stats.repos++; > + entityq_flush(&q, &rt.repos[i]); > + } > + } > + > /* > * The parser has finished something for us. > * Dequeue these one by one. > @@ -1072,6 +1225,7 @@ main(int argc, char *argv[]) > > close(proc); > close(rsync); > + close(http); > > if (waitpid(procpid, &st, 0) == -1) > err(1, "waitpid"); > @@ -1086,6 +1240,13 @@ main(int argc, char *argv[]) > warnx("rsync process exited abnormally"); > rc = 1; > } > + > + if (waitpid(httppid, &st, 0) == -1) > + err(1, "waitpid"); > + if (!WIFEXITED(st) || WEXITSTATUS(st) != 0) { > + warnx("http process exited abnormally"); > + rc = 1; > + } > } > gettimeofday(&now_time, NULL); > timersub(&now_time, &start_time, &stats.elapsed_time); > @@ -1120,6 +1281,7 @@ main(int argc, char *argv[]) > for (i = 0; i < rt.reposz; i++) { > free(rt.repos[i].repouri); > free(rt.repos[i].local); > + free(rt.repos[i].temp); > free(rt.repos[i].uris[0]); > free(rt.repos[i].uris[1]); > } > Index: mkostempat.c > =================================================================== > RCS file: mkostempat.c > diff -N mkostempat.c > --- /dev/null 1 Jan 1970 00:00:00 -0000 > +++ mkostempat.c 19 Feb 2021 14:11:28 -0000 > @@ -0,0 +1,94 @@ > +/* $OpenBSD$ */ > +/* > + * Copyright (c) 1996-1998, 2008 Theo de Raadt > + * Copyright (c) 1997, 2008-2009 Todd C. Miller > + * > + * Permission to use, copy, modify, and distribute this software for any > + * purpose with or without fee is hereby granted, provided that the above > + * copyright notice and this permission notice appear in all copies. > + * > + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES > + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF > + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR > + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES > + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN > + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF > + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. > + */ > + > +#include <sys/types.h> > +#include <sys/stat.h> > +#include <errno.h> > +#include <fcntl.h> > +#include <limits.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <string.h> > +#include <ctype.h> > +#include <unistd.h> > + > +#include "extern.h" > + > +#define TEMPCHARS > "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" > +#define NUM_CHARS (sizeof(TEMPCHARS) - 1) > +#define MIN_X 6 > + > +#define MKOTEMP_FLAGS (O_APPEND | O_CLOEXEC | O_DSYNC | O_RSYNC | > O_SYNC) > + > +#ifndef nitems > +#define nitems(_a) (sizeof((_a)) / sizeof((_a)[0])) > +#endif > + > +int > +mkostempat(int dfd, char *path, int flags) > +{ > + char *start, *cp, *ep; > + const char tempchars[] = TEMPCHARS; > + unsigned int tries; > + size_t len; > + int fd; > + > + len = strlen(path); > + if (len < MIN_X) { > + errno = EINVAL; > + return(-1); > + } > + ep = path + len; > + > + for (start = ep; start > path && start[-1] == 'X'; start--) > + ; > + if (ep - start < MIN_X) { > + errno = EINVAL; > + return(-1); > + } > + > + if (flags & ~MKOTEMP_FLAGS) { > + errno = EINVAL; > + return(-1); > + } > + flags |= O_CREAT | O_EXCL | O_RDWR; > + > + tries = INT_MAX; > + do { > + cp = start; > + do { > + unsigned short rbuf[16]; > + unsigned int i; > + > + /* > + * Avoid lots of arc4random() calls by using > + * a buffer sized for up to 16 Xs at a time. > + */ > + arc4random_buf(rbuf, sizeof(rbuf)); > + for (i = 0; i < nitems(rbuf) && cp != ep; i++) > + *cp++ = tempchars[rbuf[i] % NUM_CHARS]; > + } while (cp != ep); > + > + fd = openat(dfd, path, flags, S_IRUSR|S_IWUSR); > + if (fd != -1 || errno != EEXIST) > + return(fd); > + } while (--tries); > + > + errno = EEXIST; > + return(-1); > +} >