On Sun, Feb 28, 2021 at 09:09:05AM +0100, Theo Buehler wrote:
> On Thu, Feb 25, 2021 at 05:03:19PM +0100, Claudio Jeker wrote:
> > On Fri, Feb 19, 2021 at 07:10:02PM +0100, Claudio Jeker wrote:
> > > Some TAL files now include an https URI where the TA can be fetched from.
> > > With this diff rpki-client will download the TA from https unless that
> > > fails and then fall back to rsync.
> > >
> > > This is not yet perfect but the diff is already large enough (adding a
> > > full event based https client based on ftp codebase). For RRDP more a lot
> > > more is required and I probably will refactor the main.c code then.
> > >
> > > At the moment this adds a local mkostempat() function to implement
> > > mkostemp() but with openat() instead of open(). I hope in the future libc
> > > will provide something.
> > >
> > > Thanks in advance for all the feedback
> >
> > Updated diff. I found some bugs in the http.c code base regarding
> > conncetion failures (because of unreachable IPv6 on the server) and fixed
> > redirects. Those should now be fixed.
>
> A couple of comments inline. Some cosmetic, some minor bugs.
>
> Apart from a return value glitch for tls_close() the libtls handling
> looks correct to me and while I'm not a http expert, the state machine
> makes sense. The string handling (chopping up and re-encoding) looks
> correct with appropriate bounds checking. I spent quite a bit of time
> going through it.
>
> Take from my comments whatever you like/makes sense, but I'm ok with
> landing this and improving in tree.
Thanks a lot for the review. I know that the code is a bit of mish mash
mainly since most of it was stolen from ftp(1).
> > Index: http.c
> > ===================================================================
> > RCS file: http.c
> > diff -N http.c
> > --- /dev/null 1 Jan 1970 00:00:00 -0000
> > +++ http.c 24 Feb 2021 20:03:25 -0000
> > @@ -0,0 +1,1226 @@
> > +/* $OpenBSD$ */
> > +/*
> > + * Copyright (c) 2020 Nils Fisher <nils_fis...@hotmail.com>
> > + * Copyright (c) 2020 Claudio Jeker <clau...@openbsd.com>
> > + *
> > + * Permission to use, copy, modify, and distribute this software for any
> > + * purpose with or without fee is hereby granted, provided that the above
> > + * copyright notice and this permission notice appear in all copies.
> > + *
> > + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> > + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> > + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> > + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> > + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> > + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> > + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> > + */
> > +
> > +/*-
> > + * Copyright (c) 1997 The NetBSD Foundation, Inc.
> > + * All rights reserved.
> > + *
> > + * This code is derived from software contributed to The NetBSD Foundation
> > + * by Jason Thorpe and Luke Mewburn.
> > + *
> > + * Redistribution and use in source and binary forms, with or without
> > + * modification, are permitted provided that the following conditions
> > + * are met:
> > + * 1. Redistributions of source code must retain the above copyright
> > + * notice, this list of conditions and the following disclaimer.
> > + * 2. Redistributions in binary form must reproduce the above copyright
> > + * notice, this list of conditions and the following disclaimer in the
> > + * documentation and/or other materials provided with the distribution.
> > + *
> > + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
> > CONTRIBUTORS
> > + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> > LIMITED
> > + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
> > PARTICULAR
> > + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR
> > CONTRIBUTORS
> > + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
> > + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
> > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
> > + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
> > + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
> > + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
> > THE
> > + * POSSIBILITY OF SUCH DAMAGE.
> > + */
> > +#include <sys/types.h>
> > +#include <sys/queue.h>
> > +#include <sys/socket.h>
> > +
> > +#include <ctype.h>
> > +#include <err.h>
> > +#include <errno.h>
> > +#include <limits.h>
> > +#include <netdb.h>
> > +#include <poll.h>
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <unistd.h>
> > +#include <vis.h>
> > +#include <imsg.h>
> > +
> > +#include <tls.h>
> > +
> > +#include "extern.h"
> > +
> > +#define HTTP_USER_AGENT "OpenBSD rpki-client"
> > +#define HTTP_BUF_SIZE (32 * 1024)
> > +#define MAX_CONNECTIONS 12
> > +
> > +#define WANT_POLLIN 1
> > +#define WANT_POLLOUT 2
> > +
> > +enum http_state {
> > + STATE_FREE,
> > + STATE_INIT,
> > + STATE_CONNECT,
> > + STATE_TLSCONNECT,
> > + STATE_REQUEST,
> > + STATE_RESPONSE_STATUS,
> > + STATE_RESPONSE_HEADER,
> > + STATE_RESPONSE_DATA,
> > + STATE_RESPONSE_CHUNKED,
> > + STATE_WRITE_DATA,
> > + STATE_DONE,
> > +};
> > +
> > +struct http_proxy {
> > + char *proxyhost;
> > + char *proxyuser;
> > + char *proxypw;
> > +};
> > +
> > +struct http_connection {
> > + char *url;
> > + char *host;
> > + char *port;
> > + char *path; /* points into url */
>
> maybe make path const so it's clear that it's not meant for freeing?
Makes sense and does not any compile issues so I changed it.
> > + char *modified_since;
> > + char *last_modified;
> > + struct addrinfo *res0;
> > + struct addrinfo *res;
> > + struct tls *tls;
> > + char *buf;
> > + size_t bufsz;
> > + size_t bufpos;
> > + size_t id;
> > + size_t chunksz;
> > + off_t filesize;
> > + off_t bytes;
> > + int status;
> > + int redirect_loop;
> > + int fd;
> > + int outfd;
> > + short events;
> > + short chunked;
> > + enum http_state state;
> > +};
> > +
> > +struct msgbuf msgq;
> > +struct sockaddr_storage http_bindaddr;
> > +struct tls_config *tls_config;
> > +uint8_t *tls_ca_mem;
> > +size_t tls_ca_size;
> > +
> > +char *resp_buf[MAX_CONNECTIONS];
> > +size_t resp_bsz[MAX_CONNECTIONS];
> > +size_t resp_idx;
> > +
> > +/*
> > + * Return a string that can be used in error message to identify the
> > + * connection.
> > + */
> > +static const char *
> > +http_info(const char *url)
> > +{
> > + static char buf[64];
> > +
> > + if (strnvis(buf, url, sizeof buf, VIS_SAFE) >= (int)sizeof buf) {
> > + /* overflow, add indicator */
> > + memcpy(buf + sizeof buf - 4, "...", 4);
> > + }
> > +
> > + return buf;
> > +}
> > +
> > +/*
> > + * Determine whether the character needs encoding, per RFC1738:
> > + * - No corresponding graphic US-ASCII.
> > + * - Unsafe characters.
> > + */
> > +static int
> > +unsafe_char(const char *c0)
> > +{
> > + const char *unsafe_chars = " <>\"#{}|\\^~[]`";
> > + const unsigned char *c = (const unsigned char *)c0;
> > +
> > + /*
> > + * No corresponding graphic US-ASCII.
> > + * Control characters and octets not used in US-ASCII.
> > + */
> > + return (iscntrl(*c) || !isascii(*c) ||
> > +
> > + /*
> > + * Unsafe characters.
> > + * '%' is also unsafe, if is not followed by two
> > + * hexadecimal digits.
> > + */
> > + strchr(unsafe_chars, *c) != NULL ||
> > + (*c == '%' && (!isxdigit(*++c) || !isxdigit(*++c))));
> > +}
> > +
> > +/*
> > + * Encode given URL, per RFC1738.
> > + * Allocate and return string to the caller.
> > + */
> > +static char *
> > +url_encode(const char *path)
> > +{
> > + size_t i, length, new_length;
> > + char *epath, *epathp;
> > +
> > + length = new_length = strlen(path);
> > +
> > + /*
> > + * First pass:
> > + * Count unsafe characters, and determine length of the
> > + * final URL.
> > + */
> > + for (i = 0; i < length; i++)
> > + if (unsafe_char(path + i))
> > + new_length += 2;
> > +
> > + epath = epathp = malloc(new_length + 1); /* One more for '\0'. */
> > + if (epath == NULL)
> > + err(1, NULL);
> > +
> > + /*
> > + * Second pass:
> > + * Encode, and copy final URL.
> > + */
> > + for (i = 0; i < length; i++)
> > + if (unsafe_char(path + i)) {
> > + snprintf(epathp, 4, "%%" "%02x",
> > + (unsigned char)path[i]);
> > + epathp += 3;
> > + } else
> > + *(epathp++) = path[i];
> > +
> > + *epathp = '\0';
> > + return (epath);
> > +}
> > +
> > +static void
> > +http_setup(void)
> > +{
> > + tls_config = tls_config_new();
> > + if (tls_config == NULL)
> > + errx(1, "tls config failed");
> > +#if 0
> > + /* TODO Should we allow extra protos and ciphers? */
> > + if (tls_config_set_protocols(tls_config,
> > + TLS_PROTOCOLS_ALL) != 0)
>
> libtls returns -1 or NULL on error, so I would check for -1.
> I won't annotate this further.
I change the error check to -1. This was stolen from ftp(1) so maybe this
should be fixed there as well.
I changed to == -1 here and in all other places.
> > + errx(1, "tls set protocols failed: %s",
> > + tls_config_error(tls_config));
> > + if (tls_config_set_ciphers(tls_config, "legacy") != 0)
> > + errx(1, "tls set ciphers failed: %s",
> > + tls_config_error(tls_config));
> > +#endif
> > +
> > + /* load cert file from disk now */
> > + tls_ca_mem = tls_load_file(tls_default_ca_cert_file(),
> > + &tls_ca_size, NULL);
> > + if (tls_ca_mem == NULL)
> > + err(1, "tls_load_file");
> > + tls_config_set_ca_mem(tls_config, tls_ca_mem, tls_ca_size);
> > +
> > + /* TODO initalize proxy settings */
> > +
> > +}
> > +
> > +static int
> > +http_resolv(struct http_connection *conn, const char *host, const char
> > *port)
> > +{
> > + struct addrinfo hints;
> > + int error;
> > +
> > + memset(&hints, 0, sizeof(hints));
> > + hints.ai_family = PF_UNSPEC;
> > + hints.ai_socktype = SOCK_STREAM;
> > + error = getaddrinfo(host, port, &hints, &conn->res0);
> > + /*
> > + * If the services file is corrupt/missing, fall back
> > + * on our hard-coded defines.
> > + */
> > + if (error == EAI_SERVICE)
> > + error = getaddrinfo(host, "443", &hints, &conn->res0);
> > + if (error) {
>
> error != NULL
>
> > + warnx("%s: %s", host, gai_strerror(error));
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static void
> > +http_done(struct http_connection *conn, int ok)
> > +{
> > + struct ibuf *b;
> > +
> > + conn->state = STATE_DONE;
> > +
> > + if ((b = ibuf_dynamic(64, UINT_MAX)) == NULL)
> > + err(1, NULL);
> > + io_simple_buffer(b, &conn->id, sizeof(conn->id));
> > + io_simple_buffer(b, &ok, sizeof(ok));
> > +#if 0 /* TODO: cache last_modified */
> > + io_str_buffer(b, conn->last_modified);
> > +#endif
> > + ibuf_close(&msgq, b);
> > +}
> > +
> > +static void
> > +http_fail(size_t id)
> > +{
> > + struct ibuf *b;
> > + int ok = 0;
> > +
> > + if ((b = ibuf_dynamic(8, UINT_MAX)) == NULL)
> > + err(1, NULL);
> > + io_simple_buffer(b, &id, sizeof(id));
> > + io_simple_buffer(b, &ok, sizeof(ok));
> > + ibuf_close(&msgq, b);
> > +}
> > +
> > +static void
> > +http_free(struct http_connection *conn)
> > +{
> > + if (conn->state != STATE_DONE)
> > + http_fail(conn->id);
> > +
> > + free(conn->host);
> > + free(conn->port);
> > + free(conn->url);
>
> I would move free(conn->url) to the top so you free stuff in the order of the
> struct members.
>
Changed
> > + /* no need to free conn->path it points into conn->uri */
>
> typo: conn->url
Fixed
> > + free(conn->modified_since);
> > + free(conn->last_modified);
> > + free(conn->buf);
> > +
> > + if (conn->res0)
> > + freeaddrinfo(conn->res0);
> > +
> > + tls_free(conn->tls);
> > + conn->tls = NULL;
>
> No need for setting this to NULL.
Good point, I guess that was copy paste from the redirect code path.
> > +
> > + if (conn->fd != -1)
> > + close(conn->fd);
> > + close(conn->outfd);
> > + free(conn);
> > +}
> > +
> > +static int
> > +http_close(struct http_connection *conn)
> > +{
> > + if (conn->tls) {
>
> The file is a bit inconsistent with comparisons to NULL.
> I would prefer if they were done all explicitly but will stop annotating
> this here.
Yeah, I tried to change to the explicit != NULL check.
> > + switch (tls_close(conn->tls)) {
> > + case TLS_WANT_POLLIN:
> > + return WANT_POLLIN;
> > + case TLS_WANT_POLLOUT:
> > + return WANT_POLLOUT;
>
> Looks like http_close() failure is currently silent, otherwise the missing
> success case would have been noticed:
>
> case 0:
> return 0;
This actually needs to be return -1. I added an explicit case 0 but
changed it to just break since we want to return -1 so that the connection
is freed in http_do(). If 0 is returned then tls_close would be called
more than once.
> > + case -1:
> > + warnx("%s: TLS close: %s", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > + }
> > + }
> > +
> > + return -1;
> > +}
> > +
> > +static int
> > +http_parse_uri(char *uri, char **ohost, char **oport, char **opath)
> > +{
> > + char *host, *port = NULL, *path;
> > + char *hosttail;
> > +
> > + if (strncasecmp(uri, "https://";, 8) != 0) {
> > + warnx("%s: not using https schema", http_info(uri));
> > + return -1;
> > + }
> > + host = uri + 8;
> > + if ((path = strchr(host, '/')) == NULL) {
> > + warnx("%s: missing https path", http_info(uri));
> > + return -1;
> > + }
> > + if (path - uri > INT_MAX - 1) {
> > + warnx("%s: preposterous host length", http_info(uri));
> > + return -1;
> > + }
> > + if (*host == '[') {
> > + char *scope;
> > + if ((hosttail = memrchr(host, ']', path - host)) != NULL &&
> > + (hosttail[1] == '/' || hosttail[1] == ':'))
> > + host++;
> > + if (hosttail[1] == ':')
> > + port = hosttail + 1;
> > + if ((scope = memchr(host, '%', hosttail - host)) != NULL)
> > + hosttail = scope;
> > + } else {
> > + if ((hosttail = memrchr(host, ':', path - host)) != NULL)
> > + port = hosttail + 1;
> > + else
> > + hosttail = path;
> > + }
> > +
> > + if ((host = strndup(host, hosttail - host)) == NULL)
> > + err(1, "strndup");
> > + if (port) {
> > + if ((port = strndup(port, path - port)) == NULL)
> > + err(1, "strndup");
> > + } else {
> > + if ((port = strdup("443")) == NULL)
> > + err(1, "strdup");
> > + }
> > + /* do not include the initial / in path */
> > + path++;
> > +
> > + *ohost = host;
> > + *oport = port;
> > + *opath = path;
> > +
> > + return 0;
> > +}
> > +
> > +
> > +static struct http_connection *
> > +http_new(size_t id, char *uri, char *modified_since, int outfd)
> > +{
> > + struct http_connection *conn;
> > + char *host, *port, *path;
> > +
> > + if (http_parse_uri(uri, &host, &port, &path) == -1) {
> > + free(uri);
> > + free(modified_since);
> > + return NULL;
> > + }
> > +
> > + if ((conn = calloc(1, sizeof(*conn))) == NULL)
> > + err(1, NULL);
> > +
> > + conn->id = id;
> > + conn->fd = -1;
> > + conn->outfd = outfd;
> > + conn->host = host;
> > + conn->port = port;
> > + conn->path = path;
> > + conn->url = uri;
> > + conn->modified_since = modified_since;
> > + conn->state = STATE_INIT;
> > +
> > + /* TODO proxy support (overload of host and port) */
> > +
> > + if (http_resolv(conn, host, port) == -1) {
> > + http_free(conn);
> > + return NULL;
> > + }
> > +
> > + return conn;
> > +}
> > +
> > +static int
> > +http_redirect(struct http_connection *conn, char *uri)
> > +{
> > + char *host, *port, *path;
> > +
> > + warnx("redirect to %s", http_info(uri));
> > +
> > + if (http_parse_uri(uri, &host, &port, &path) == -1) {
> > + free(uri);
> > + return -1;
> > + }
> > +
> > + free(conn->host);
> > + conn->host = host;
> > + free(conn->port);
> > + conn->port = port;
> > + free(conn->url);
> > + conn->url = uri;
>
> move handling of url to the beginning
Done.
> > + conn->path = path;
>
> conn->modified_since omitted?
Yes, we should include the same modified_since header in the redirect so
that the next server is able to do the same check and return a 304 if
nothing got changed.
> > + free(conn->last_modified);
> > + conn->last_modified = NULL;
> > + free(conn->buf);
> > + conn->buf = NULL;
> > + conn->bufpos = 0;
> > + conn->bufsz = 0;
> > + tls_close(conn->tls);
> > + tls_free(conn->tls);
> > + conn->tls = NULL;
> > + close(conn->fd);
> > + conn->state = STATE_INIT;
> > +
> > + /* TODO proxy support (overload of host and port) */
> > +
> > + if (http_resolv(conn, host, port) == -1)
> > + return -1;
> > +
> > + return 0;
> > +}
> > +
> > +static int
> > +http_connect(struct http_connection *conn)
> > +{
> > + char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST];
> > + char *cause = "unknown";
> > +
> > + if (conn->fd != -1) {
> > + close(conn->fd);
> > + conn->fd = -1;
> > + }
> > +
> > + /* start the loop below with first or next address */
> > + if (conn->res == NULL)
> > + conn->res = conn->res0;
> > + else
> > + conn->res = conn->res->ai_next;
> > + for (; conn->res != NULL; conn->res = conn->res->ai_next) {
> > + struct addrinfo *res = conn->res;
> > + int fd, error, save_errno;
> > +
> > + if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf,
> > + sizeof(hbuf), pbuf, sizeof(pbuf),
> > + NI_NUMERICHOST | NI_NUMERICSERV) != 0)
> > + strlcpy(hbuf, "(unknown)", sizeof(hbuf));
> > +
> > + fd = socket(res->ai_family,
> > + res->ai_socktype | SOCK_NONBLOCK, res->ai_protocol);
> > + if (fd == -1) {
> > + cause = "socket";
> > + continue;
> > + }
> > + conn->fd = fd;
> > +
> > + if (http_bindaddr.ss_family == res->ai_family) {
> > + if (bind(conn->fd, (struct sockaddr *)&http_bindaddr,
> > + res->ai_addrlen) == -1)
> > + warn("%s: bind", http_info(conn->url));
> > + }
> > +
> > + error = connect(conn->fd, res->ai_addr, res->ai_addrlen);
> > + if (error == -1) {
> > + if (errno == EINPROGRESS) {
> > + /* waiting for connect to finish. */
> > + return WANT_POLLOUT;
> > + } else {
> > + save_errno = errno;
> > + close(conn->fd);
> > + conn->fd = -1;
> > + errno = save_errno;
> > + cause = "connect";
> > + continue;
> > + }
> > + }
> > +
> > + /* TODO proxy connect */
> > +#if 0
> > + if (proxyenv)
> > + proxy_connect(conn->fd, sslhost, proxy_credentials); */
> > +#endif
> > + }
> > + freeaddrinfo(conn->res0);
> > + conn->res0 = NULL;
> > + if (conn->fd == -1) {
> > + warn("%s: %s", http_info(conn->url), cause);
> > + return -1;
> > + }
> > + return 0;
> > +}
> > +
> > +static int
> > +http_finish_connect(struct http_connection *conn)
> > +{
> > + int error = 0;
> > + socklen_t len;
> > +
> > + len = sizeof(error);
> > + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &error, &len) == -1) {
> > + warn("%s: getsockopt SO_ERROR", http_info(conn->url));
> > + /* connection will be closed by http_connect() */
> > + return -1;
> > + }
> > + if (error != 0) {
> > + errno = error;
> > + warn("%s: connect", http_info(conn->url));
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static int
> > +http_tls_handshake(struct http_connection *conn)
> > +{
> > + switch (tls_handshake(conn->tls)) {
> > + case 0:
> > + return 0;
> > + case TLS_WANT_POLLIN:
> > + return WANT_POLLIN;
> > + case TLS_WANT_POLLOUT:
> > + return WANT_POLLOUT;
> > + }
> > + warnx("%s: TLS handshake: %s", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > +}
> > +
> > +static int
> > +http_tls_connect(struct http_connection *conn)
> > +{
> > + if ((conn->tls = tls_client()) == NULL) {
> > + warn("tls_client");
> > + return -1;
> > + }
> > + if (tls_configure(conn->tls, tls_config) != 0) {
> > + warnx("%s: TLS configuration: %s\n", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > + }
> > + if (tls_connect_socket(conn->tls, conn->fd, conn->host) != 0) {
> > + warnx("%s: TLS connect: %s\n", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > + }
> > + return http_tls_handshake(conn);
> > +}
> > +
> > +static int
> > +http_request(struct http_connection *conn)
> > +{
> > + char *host, *epath, *modified_since;
> > + int with_port = 0;
> > +
> > + /* TODO adjust request for HTTP proxy setups */
> > +
> > + /*
> > + * Send port number only if it's specified and does not equal
> > + * the default. Some broken HTTP servers get confused if you explicitly
> > + * send them the port number.
> > + */
> > + if (conn->port && strcmp(conn->port, "443") != 0)
> > + with_port = 1;
> > +
> > + /* Construct the Host header from host and port info */
> > + if (strchr(conn->host, ':')) {
> > + if (asprintf(&host, "[%s]%s%s", conn->host,
> > + with_port ? ":" : "", with_port ? conn->port : "") == -1)
> > + err(1, NULL);
> > +
> > + } else {
> > + if (asprintf(&host, "%s%s%s", conn->host,
> > + with_port ? ":" : "", with_port ? conn->port : "") == -1)
> > + err(1, NULL);
> > + }
> > +
> > + /*
> > + * Construct and send the request. Proxy requests don't want leading /.
> > + */
> > + epath = url_encode(conn->path);
> > +
> > + modified_since = NULL;
> > + if (conn->modified_since) {
> > + if (asprintf(&modified_since, "If-Modified-Since: %s\r\n",
> > + conn->modified_since) == -1)
> > + err(1, NULL);
> > + }
> > +
> > + free(conn->buf);
> > + conn->bufpos = 0;
> > + if ((conn->bufsz = asprintf(&conn->buf,
> > + "GET /%s HTTP/1.1\r\n"
> > + "Connection: close\r\n"
> > + "User-Agent: " HTTP_USER_AGENT "\r\n"
> > + "Host: %s\r\n%s\r\n",
> > + epath, host,
> > + modified_since ? modified_since : "")) == -1)
> > + err(1, NULL);
> > +
> > + free(epath);
> > + free(host);
> > + free(modified_since);
> > +
> > + return WANT_POLLOUT;
> > +}
> > +
> > +static int
> > +http_write(struct http_connection *conn)
> > +{
> > + ssize_t s;
> > +
> > + s = tls_write(conn->tls, conn->buf + conn->bufpos,
> > + conn->bufsz - conn->bufpos);
> > + if (s == -1) {
> > + warnx("%s: TLS write: %s", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > + } else if (s == TLS_WANT_POLLIN) {
> > + return WANT_POLLIN;
> > + } else if (s == TLS_WANT_POLLOUT) {
> > + return WANT_POLLOUT;
> > + }
> > +
> > + conn->bufpos += s;
> > + if (conn->bufpos == conn->bufsz)
> > + return 0;
> > + return WANT_POLLOUT;
> > +}
> > +
> > +static int
> > +http_parse_status(struct http_connection *conn, char *buf)
> > +{
> > + const char *errstr;
> > + char *cp, ststr[4];
> > + char gerror[200];
> > + int status;
> > +
> > + cp = strchr(buf, ' ');
> > + if (cp == NULL) {
> > + warnx("Improper response from %s", http_info(conn->url));
> > + return -1;
> > + } else
> > + cp++;
> > +
> > + strlcpy(ststr, cp, sizeof(ststr));
> > + status = strtonum(ststr, 200, 599, &errstr);
> > + if (errstr) {
> > + strnvis(gerror, cp, sizeof gerror, VIS_SAFE);
> > + warnx("Error retrieving %s: %s", http_info(conn->url), gerror);
> > + return -1;
> > + }
> > +
> > + switch (status) {
> > + case 301:
> > + case 302:
> > + case 303:
> > + case 307:
> > + if (conn->redirect_loop++ > 10) {
> > + warnx("%s: Too many redirections requested",
> > + http_info(conn->url));
> > + return -1;
> > + }
> > + /* FALLTHROUGH */
> > + case 200:
> > + case 206:
> > + case 304:
> > + conn->status = status;
> > + break;
> > + default:
> > + strnvis(gerror, cp, sizeof gerror, VIS_SAFE);
> > + warnx("Error retrieving %s: %s", http_info(conn->url), gerror);
> > + break;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static inline int
> > +http_isredirect(struct http_connection *conn)
> > +{
> > + if ((conn->status >= 301 && conn->status <= 303) ||
> > + conn->status == 307)
> > + return 1;
> > + return 0;
> > +}
> > +
> > +static int
> > +http_parse_header(struct http_connection *conn, char *buf)
> > +{
> > +#define CONTENTLEN "Content-Length: "
> > +#define LOCATION "Location: "
> > +#define TRANSFER_ENCODING "Transfer-Encoding: "
> > +#define LAST_MODIFIED "Last-Modified: "
> > + const char *errstr;
> > + char *cp, *redirurl;
> > + char *locbase, *loctail;
> > +
> > + cp = buf;
> > + /* empty line, end of header */
> > + if (*cp == '\0')
> > + return 0;
> > + else if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) {
> > + size_t s;
> > + cp += sizeof(CONTENTLEN) - 1;
> > + if ((s = strcspn(cp, " \t")))
> > + *(cp+s) = 0;
> > + conn->filesize = strtonum(cp, 0, LLONG_MAX, &errstr);
> > + if (errstr != NULL) {
> > + warnx("Improper response from %s",
> > + http_info(conn->url));
> > + return -1;
> > + }
> > + } else if (http_isredirect(conn) &&
> > + strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) {
> > + cp += sizeof(LOCATION) - 1;
> > + /*
> > + * If there is a colon before the first slash, this URI
> > + * is not relative. RFC 3986 4.2
> > + */
> > + if (cp[strcspn(cp, ":/")] != ':') {
> > + /* XXX doesn't handle protocol-relative URIs */
> > + if (*cp == '/') {
> > + locbase = NULL;
> > + cp++;
> > + } else {
> > + locbase = strdup(conn->path);
> > + if (locbase == NULL)
> > + err(1, "");
> > + loctail = strchr(locbase, '#');
> > + if (loctail != NULL)
> > + *loctail = '\0';
> > + loctail = strchr(locbase, '?');
> > + if (loctail != NULL)
> > + *loctail = '\0';
> > + loctail = strrchr(locbase, '/');
> > + if (loctail == NULL) {
> > + free(locbase);
> > + locbase = NULL;
> > + } else
> > + loctail[1] = '\0';
> > + }
> > + /* Contruct URL from relative redirect */
>
> Typo: Construct
>
> > + if (asprintf(&redirurl, "%.*s/%s%s",
> > + (int)(conn->path - conn->url), conn->url,
> > + locbase ? locbase : "",
> > + cp) == -1)
> > + err(1, "Cannot build redirect URL");
> > + free(locbase);
> > + } else if ((redirurl = strdup(cp)) == NULL)
> > + err(1, "Cannot build redirect URL");
> > + loctail = strchr(redirurl, '#');
> > + if (loctail != NULL)
> > + *loctail = '\0';
> > + return http_redirect(conn, redirurl);
> > + } else if (strncasecmp(cp, TRANSFER_ENCODING,
> > + sizeof(TRANSFER_ENCODING) - 1) == 0) {
> > + cp += sizeof(TRANSFER_ENCODING) - 1;
> > + cp[strcspn(cp, " \t")] = '\0';
> > + if (strcasecmp(cp, "chunked") == 0)
> > + conn->chunked = 1;
> > + } else if (strncasecmp(cp, LAST_MODIFIED,
> > + sizeof(LAST_MODIFIED) - 1) == 0) {
> > + cp += sizeof(LAST_MODIFIED) - 1;
> > + conn->last_modified = strdup(cp);
> > + }
> > +
> > + return 1;
> > +}
> > +
> > +static char *
> > +http_get_line(struct http_connection *conn)
> > +{
> > + char *end, *line;
> > + size_t len;
> > +
> > + end = memchr(conn->buf, '\n', conn->bufpos);
> > + if (end == NULL)
> > + return NULL;
> > +
> > + len = end - conn->buf;
> > + while (len > 0 && conn->buf[len - 1] == '\r')
> > + --len;
> > + line = strndup(conn->buf, len);
>
> Should strndup failure not be a fatal error?
Absolutly. Fixed that.
> > +
> > + /* consume line including \n */
> > + end++;
> > + conn->bufpos -= end - conn->buf;
> > + memmove(conn->buf, end, conn->bufpos);
> > +
> > + return line;
> > +}
> > +
> > +static int
> > +http_parse_chunked(struct http_connection *conn, char *buf)
> > +{
> > + char *header = buf;
> > + char *end;
> > + int chunksize;
> > +
> > + /* ignore empty lines, used between chunk and next header */
> > + if (*header == '\0')
> > + return 1;
> > +
> > + /* strip CRLF and any optional chunk extension */
> > + header[strcspn(header, ";\r\n")] = '\0';
> > + errno = 0;
> > + chunksize = strtoul(header, &end, 16);
> > + if (errno || header[0] == '\0' || *end != '\0' ||
> > + chunksize > INT_MAX) {
> > + warnx("%s: Invalid chunk size", http_info(conn->url));
> > + return -1;
> > + }
> > + conn->chunksz = chunksize;
> > +
> > + if (conn->chunksz == 0) {
> > + http_done(conn, 1);
> > + return 0;
> > + }
> > +
> > + return 1;
> > +}
> > +
> > +static int
> > +http_read(struct http_connection *conn)
> > +{
> > + ssize_t s;
> > + char *buf;
> > +
> > + s = tls_read(conn->tls, conn->buf + conn->bufpos,
> > + conn->bufsz - conn->bufpos);
> > + if (s == -1) {
> > + warn("%s: TLS read: %s", http_info(conn->url),
> > + tls_error(conn->tls));
> > + return -1;
> > + } else if (s == TLS_WANT_POLLIN) {
> > + return WANT_POLLIN;
> > + } else if (s == TLS_WANT_POLLOUT) {
> > + return WANT_POLLOUT;
> > + }
> > +
> > + if (s == 0 && conn->bufpos == 0) {
> > + warnx("%s: short read, connection closed",
> > + http_info(conn->url));
> > + return -1;
> > + }
> > +
> > + conn->bufpos += s;
> > +
> > + switch (conn->state) {
> > + case STATE_RESPONSE_STATUS:
> > + buf = http_get_line(conn);
> > + if (buf == NULL)
> > + return WANT_POLLIN;
> > + if (http_parse_status(conn, buf) == -1) {
> > + free(buf);
> > + return -1;
> > + }
> > + free(buf);
> > + conn->state = STATE_RESPONSE_HEADER;
> > + /* FALLTHROUGH */
> > + case STATE_RESPONSE_HEADER:
> > + while ((buf = http_get_line(conn)) != NULL) {
> > + int rv;
> > +
> > + rv = http_parse_header(conn, buf);
> > + free(buf);
> > + if (rv != 1)
> > + return rv;
> > + }
> > + return WANT_POLLIN;
> > + case STATE_RESPONSE_DATA:
> > + if (conn->bufpos == conn->bufsz ||
> > + conn->filesize - conn->bytes <= (off_t)conn->bufpos)
> > + return 0;
> > + return WANT_POLLIN;
> > + case STATE_RESPONSE_CHUNKED:
> > + while (conn->chunksz == 0) {
> > + buf = http_get_line(conn);
> > + if (buf == NULL)
> > + return WANT_POLLIN;
> > + switch (http_parse_chunked(conn, buf)) {
> > + case -1:
> > + free(buf);
> > + return -1;
> > + case 0:
> > + free(buf);
> > + return 0;
> > + }
> > + free(buf);
> > + }
> > +
> > + if (conn->bufpos == conn->bufsz ||
> > + conn->chunksz <= conn->bufpos)
> > + return 0;
> > + return WANT_POLLIN;
> > + default:
> > + errx(1, "unexpected http state");
> > + }
> > +}
> > +
> > +static int
> > +data_write(struct http_connection *conn)
> > +{
> > + ssize_t s;
> > + size_t bsz = conn->bufpos;
> > +
> > + if (conn->filesize - conn->bytes < (off_t)bsz)
> > + bsz = conn->filesize - conn->bytes;
> > +
> > + s = write(conn->outfd, conn->buf, bsz);
> > + if (s == -1) {
> > + warn("%s: Data write", http_info(conn->url));
> > + return -1;
> > + }
> > +
> > + conn->bufpos -= s;
> > + conn->bytes += s;
> > + memmove(conn->buf, conn->buf + s, conn->bufpos);
> > +
> > + if (conn->bytes == conn->filesize) {
> > + http_done(conn, 1);
> > + return 0;
> > + }
> > +
> > + if (conn->bufpos == 0)
> > + return 0;
> > +
> > + return WANT_POLLOUT;
> > +}
> > +
> > +static int
> > +chunk_write(struct http_connection *conn)
> > +{
> > + ssize_t s;
> > + size_t bsz = conn->bufpos;
> > +
> > + if (bsz > conn->chunksz)
> > + bsz = conn->chunksz;
> > +
> > + s = write(conn->outfd, conn->buf, bsz);
> > + if (s == -1) {
> > + warn("%s: Chunk write", http_info(conn->url));
> > + return -1;
> > + }
> > +
> > + conn->bufpos -= s;
> > + conn->chunksz -= s;
> > + conn->bytes += s;
> > + memmove(conn->buf, conn->buf + s, conn->bufpos);
> > +
> > + if (conn->bufpos == 0 || conn->chunksz == 0)
> > + return 0;
> > +
> > + return WANT_POLLOUT;
> > +}
> > +
> > +static int
> > +http_handle(struct http_connection *conn)
> > +{
> > + switch (conn->state) {
> > + case STATE_INIT:
> > + return 0;
> > + case STATE_CONNECT:
> > + if (http_finish_connect(conn) == -1)
> > + /* something went wrong, try other host */
> > + return http_connect(conn);
> > + return 0;
> > + case STATE_TLSCONNECT:
> > + return http_tls_handshake(conn);
> > + case STATE_REQUEST:
> > + return http_write(conn);
> > + case STATE_RESPONSE_STATUS:
> > + case STATE_RESPONSE_HEADER:
> > + case STATE_RESPONSE_DATA:
> > + case STATE_RESPONSE_CHUNKED:
> > + return http_read(conn);
> > + case STATE_WRITE_DATA:
> > + if (conn->chunked)
> > + return chunk_write(conn);
> > + else
> > + return data_write(conn);
> > + case STATE_DONE:
> > + return http_close(conn);
> > + case STATE_FREE:
> > + errx(1, "bad http state");
> > + }
> > +}
> > +
> > +static int
> > +http_nextstep(struct http_connection *conn)
> > +{
> > + switch (conn->state) {
> > + case STATE_INIT:
> > + conn->state = STATE_CONNECT;
> > + return http_connect(conn);
> > + case STATE_CONNECT:
> > + conn->state = STATE_TLSCONNECT;
> > + return http_tls_connect(conn);
> > + case STATE_TLSCONNECT:
> > + conn->state = STATE_REQUEST;
> > + return http_request(conn);
> > + case STATE_REQUEST:
> > + conn->state = STATE_RESPONSE_STATUS;
> > + free(conn->buf);
> > + /* allocate the read buffer */
> > + if ((conn->buf = malloc(HTTP_BUF_SIZE)) == NULL)
> > + err(1, NULL);
> > + conn->bufpos = 0;
> > + conn->bufsz = HTTP_BUF_SIZE;
> > + return WANT_POLLIN;
> > + case STATE_RESPONSE_STATUS:
> > + conn->state = STATE_RESPONSE_HEADER;
> > + return WANT_POLLIN;
> > + case STATE_RESPONSE_HEADER:
> > + if (conn->status == 200)
> > + conn->state = STATE_RESPONSE_DATA;
> > + else {
> > + http_done(conn, 0);
> > + return http_close(conn);
> > + }
> > + return WANT_POLLIN;
> > + case STATE_RESPONSE_DATA:
> > + conn->state = STATE_WRITE_DATA;
> > + return WANT_POLLOUT;
> > + case STATE_RESPONSE_CHUNKED:
> > + conn->state = STATE_WRITE_DATA;
> > + return WANT_POLLOUT;
> > + case STATE_WRITE_DATA:
> > + if (conn->chunked)
> > + conn->state = STATE_RESPONSE_CHUNKED;
> > + else
> > + conn->state = STATE_RESPONSE_DATA;
> > + return WANT_POLLIN;
> > + case STATE_DONE:
> > + return http_close(conn);
> > + case STATE_FREE:
> > + errx(1, "bad http state");
> > + }
> > +}
> > +
> > +static int
> > +http_do(struct http_connection *conn)
> > +{
> > + switch (http_handle(conn)) {
> > + case -1:
> > + /* connection failure */
> > + http_free(conn);
> > + return -1;
> > + case 0:
> > + switch (http_nextstep(conn)) {
> > + case WANT_POLLIN:
> > + conn->events = POLLIN;
> > + break;
> > + case WANT_POLLOUT:
> > + conn->events = POLLOUT;
> > + break;
> > + case -1:
> > + http_free(conn);
> > + return -1;
> > + }
> > + break;
> > + case WANT_POLLIN:
> > + conn->events = POLLIN;
> > + break;
> > + case WANT_POLLOUT:
> > + conn->events = POLLOUT;
> > + break;
> > + }
> > + return 0;
> > +}
> > +
> > +void
> > +proc_http(char *bind_addr, int fd)
> > +{
> > + struct http_connection *http_conns[MAX_CONNECTIONS];
> > + struct pollfd pfds[MAX_CONNECTIONS + 1];
> > + size_t i;
> > + int active_connections;
> > +
> > + if (bind_addr) {
> > + struct addrinfo hints, *res;
> > +
> > + bzero(&hints, sizeof(hints));
> > + hints.ai_family = AF_UNSPEC;
> > + hints.ai_socktype = SOCK_DGRAM; /*dummy*/
> > + hints.ai_flags = AI_NUMERICHOST;
> > + if (getaddrinfo(bind_addr, NULL, &hints, &res) == 0) {
> > + memcpy(&http_bindaddr, res->ai_addr, res->ai_addrlen);
> > + freeaddrinfo(res);
> > + }
> > + }
> > + http_setup();
> > +
> > + if (pledge("stdio inet dns recvfd", NULL) == -1)
> > + err(1, "pledge");
> > +
> > + memset(&http_conns, 0, sizeof(http_conns));
> > + memset(&pfds, 0, sizeof(pfds));
> > + pfds[MAX_CONNECTIONS].fd = fd;
> > +
> > + msgbuf_init(&msgq);
> > + msgq.fd = fd;
> > +
> > + for (;;) {
> > + active_connections = 0;
> > + for (i = 0; i < MAX_CONNECTIONS; i++) {
> > + struct http_connection *conn = http_conns[i];
> > + if (conn == NULL) {
> > + pfds[i].fd = -1;
> > + continue;
> > + }
> > + if (conn->state == STATE_WRITE_DATA)
> > + pfds[i].fd = conn->outfd;
> > + else
> > + pfds[i].fd = conn->fd;
> > +
> > + pfds[i].events = conn->events;
> > + active_connections++;
> > + }
> > + pfds[MAX_CONNECTIONS].events = 0;
> > + if (active_connections < MAX_CONNECTIONS)
> > + pfds[MAX_CONNECTIONS].events |= POLLIN;
> > + if (msgq.queued)
> > + pfds[MAX_CONNECTIONS].events |= POLLOUT;
> > +
> > + if (poll(pfds, sizeof(pfds) / sizeof(pfds[0]), INFTIM) == -1)
> > + err(1, "poll");
> > +
> > + if (pfds[MAX_CONNECTIONS].revents & POLLHUP)
> > + break;
> > +
> > + if (pfds[MAX_CONNECTIONS].revents & POLLIN) {
> > + struct http_connection *h;
> > + size_t id;
> > + int outfd;
> > + char *uri;
> > + char *mod;
> > +
> > + outfd = io_recvfd(fd, &id, sizeof(id));
> > + io_str_read(fd, &uri);
> > + io_str_read(fd, &mod);
> > +
> > + h = http_new(id, uri, mod, outfd);
> > + if (h == NULL) {
> > + close(outfd);
> > + http_fail(id);
> > + } else
> > + for (i = 0; i < MAX_CONNECTIONS; i++) {
> > + if (http_conns[i] != NULL)
> > + continue;
> > + http_conns[i] = h;
> > + if (http_do(h) == -1)
> > + http_conns[i] = NULL;
> > + break;
> > + }
> > + }
> > + if (pfds[MAX_CONNECTIONS].revents & POLLOUT) {
> > + switch (msgbuf_write(&msgq)) {
> > + case 0:
> > + errx(1, "write: connection closed");
> > + case -1:
> > + err(1, "write");
> > + }
> > + }
> > + for (i = 0; i < MAX_CONNECTIONS; i++) {
> > + struct http_connection *conn = http_conns[i];
> > +
> > + if (conn == NULL)
> > + continue;
> > + /* event not ready */
> > + if (!(pfds[i].revents & (conn->events | POLLHUP)))
> > + continue;
> > +
> > + if (http_do(conn) == -1)
> > + http_conns[i] = NULL;
> > + }
> > + }
> > +}
--
:wq Claudio
