Hi, The idea is to start with the subset of ftp(1) functionality needed by pkg_add(1):
ftp [-o output] url ... i.e., should be able to download files over HTTP(S) and FTP. This implementation works as FETCH_CMD for pkg_add(1) over HTTP(S). FTP is not yet done, but thinking of implementing just PASV and RETR commands and drop command interpreter compeletely. The SMALL variant drops HTTPS and FTP support. Comments? Note: A lot of functionality is still missing including proxy, redirects, progressmeter, UerAgent, Ranges, etc. Index: Makefile =================================================================== RCS file: Makefile diff -N Makefile --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ Makefile 21 May 2015 18:23:37 -0000 @@ -0,0 +1,19 @@ +# Define SMALL to disable https and ftp support +.if defined(SMALL) +CFLAGS+= -DSMALL +.endif + +PROG= ftp +NOMAN= +DEBUG= -g -Wall -O0 +SRCS= main.c http.c +LDADD+= -lutil +DPADD+= ${LIBUTIL} + +.ifndef SMALL +SRCS+= https.c ftp.c +LDADD+= -ltls -lssl -lcrypto +DPADD+= ${LIBTLS} ${LIBSSL} ${LIBCRYPTO} +.endif + +.include <bsd.prog.mk> Index: ftp.c =================================================================== RCS file: ftp.c diff -N ftp.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ ftp.c 21 May 2015 18:23:37 -0000 @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2015 Sunil Nimmagadda <su...@nimmagadda.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/types.h> + +#include "ftp.h" + +int ftp_connect(const char *, const char *); +int ftp_get(int, const char *, const char *, int); + +struct proto proto_ftp = { + ftp_connect, + ftp_get +}; + +int +ftp_connect(const char *host, const char *port) +{ + return (-1); +} + +int +ftp_get(int s, const char *resource, const char *fn, int flags) +{ + return (-1); +} Index: ftp.h =================================================================== RCS file: ftp.h diff -N ftp.h --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ ftp.h 21 May 2015 18:23:37 -0000 @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2015 Sunil Nimmagadda <su...@nimmagadda.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#define BUF_SIZ 1024 +#define TMPBUF_LEN 131072 +#define MAX_RETRIES 10 + +struct http_headers { + char location[BUF_SIZ]; /* Redirect Location */ + off_t c_len; /* Content-Length */ +}; + +struct proto { + int (*connect)(const char *, const char *); + int (*get)(int, const char *, const char *, int); +}; + +/* main.c */ +int host_extract(char *, char *, size_t); +int header_insert(struct http_headers *, const char *); + +/* http.c */ +int http_connect(const char *, const char *); +int http_response_code(char *); + Index: http.c =================================================================== RCS file: http.c diff -N http.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ http.c 21 May 2015 18:23:37 -0000 @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2015 Sunil Nimmagadda <su...@nimmagadda.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/types.h> +#include <sys/socket.h> + +#include <ctype.h> +#include <err.h> +#include <errno.h> +#include <fcntl.h> +#include <netdb.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <util.h> + +#include "ftp.h" + +int http_get(int, const char *, const char *, int); +char *http_getline(FILE *, size_t *); +int http_parse_headers(FILE *, struct http_headers *); +void http_200(FILE *, int); + +struct proto proto_http = { + http_connect, + http_get +}; + +int +http_connect(const char *host, const char *port) +{ + struct addrinfo hints, *res, *res0; + const char *cause = NULL; + int error, s = -1, save_errno; + + if (port == NULL) + port = "www"; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + error = getaddrinfo(host, port, &hints, &res0); + if (error) { + warnx("%s: %s", host, gai_strerror(error)); + return (-1); + } + + for (res = res0; res; res = res->ai_next) { + s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (s == -1) { + cause = "socket"; + continue; + } + + if (connect(s, res->ai_addr, res->ai_addrlen) == -1) { + cause = "connect"; + save_errno = errno; + close(s); + errno = save_errno; + s = -1; + continue; + } + + break; + } + + if (s == -1) { + warn("%s", cause); + return (-1); + } + + freeaddrinfo(res0); + return (s); +} + +int +http_get(int s, const char *url, const char *fn, int flags) +{ + struct http_headers hdrs; + FILE *fin; + char *buf, *resource; + char hostname[HOST_NAME_MAX+1]; + size_t len; + int fd, res = -1; + + if ((fin = fdopen(s, "r+")) == NULL) + err(1, "http_get: fdopen"); + + if (gethostname(hostname, sizeof(hostname)) != 0) + hostname[0] = '\0'; + + resource = strstr(url, "//"); + resource += 2; + resource = strchr(resource, '/'); + + fprintf(fin, + "GET %s HTTP/1.0\r\n" + "Host: %s\r\n" + "\r\n", + (resource) ? resource : "/", + hostname); + fflush(fin); + if ((buf = http_getline(fin, &len)) == NULL) + goto cleanup; + + if ((res = http_response_code(buf)) == 0) { + warnx("failed to get response code"); + goto cleanup; + } + + memset(&hdrs, 0, sizeof(hdrs)); + if (http_parse_headers(fin, &hdrs) != 0) + goto cleanup; + + switch (res) { + case 200: + if (strcmp(fn, "-") == 0) + fd = STDOUT_FILENO; + else if ((fd = open(fn, flags, 0666)) == -1) + err(1, "open: %s", fn); + + http_200(fin, fd); + if (fd != STDOUT_FILENO) + close(fd); + + break; + } + +cleanup: + fclose(fin); + return (res); +} + +void +http_200(FILE *fin, int out) +{ + size_t r, wlen; + ssize_t i; + char *cp; + static char *buf; + + if (buf == NULL) { + buf = malloc(TMPBUF_LEN); /* allocate once */ + if (buf == NULL) + err(1, "malloc"); + } + + /* XXX Content-Length or till EOF? */ + while ((r = fread(buf, sizeof(char), TMPBUF_LEN, fin)) > 0) { + for (cp = buf, wlen = r; wlen > 0; wlen -= i, cp += i) { + if ((i = write(out, cp, wlen)) == -1) { + warn("http_200: write"); + break; + } else if (i == 0) + break; + } + } +} + +int +http_response_code(char *buf) +{ + const char *errstr; + char *e; + int res; + + if ((buf = strchr(buf, ' ')) == NULL) + return (0); + + buf++; + if ((e = strchr(buf, ' ')) == NULL) + return (0); + + *e = '\0'; + res = strtonum(buf, 200, 416, &errstr); + if (errstr) { + warn("http_response_code: strtonum"); + return (0); + } + + return (res); +} + +int +http_parse_headers(FILE *fin, struct http_headers *hdrs) +{ + char *buf; + size_t len; + + while ((buf = http_getline(fin, &len))) { + if (len == 0) + break; /* end of headers */ + + if (header_insert(hdrs, buf) != 0) + return (-1); + } + + return (0); +} + +char * +http_getline(FILE *fp, size_t *len) +{ + char *buf; + size_t ln; + + if ((buf = fparseln(fp, &ln, NULL, "\0\0\0", 0)) == NULL) + return (NULL); + + if (buf[ln - 1] == '\r') + buf[--ln] = '\0'; + + *len = ln; + return (buf); +} + Index: https.c =================================================================== RCS file: https.c diff -N https.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ https.c 21 May 2015 18:23:37 -0000 @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2015 Sunil Nimmagadda <su...@nimmagadda.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <sys/types.h> + +#include <err.h> +#include <fcntl.h> +#include <limits.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <tls.h> +#include <unistd.h> + +#include "ftp.h" + +struct tls *ctx; +struct tls_config *tls_config = NULL; + +int https_connect(const char *, const char *); +int https_get(int, const char *, const char *, int); +int https_vprintf(struct tls *, const char *, ...); +char *https_getline(struct tls *, size_t *); +int https_parse_headers(struct tls *, struct http_headers *); +void https_200(struct tls *, int); + +struct proto proto_https = { + https_connect, + https_get +}; + +int +https_connect(const char *host, const char *port) +{ + int s; + + if (tls_init() != 0) { + warnx("tls init failed"); + return (-1); + } + + if (tls_config == NULL) { + tls_config = tls_config_new(); + if (tls_config == NULL) + errx(1, "tls config failed"); + + tls_config_set_protocols(tls_config, TLS_PROTOCOLS_ALL); + if (tls_config_set_ciphers(tls_config, "compat") != 0) + errx(1, "tls set ciphers failed"); + } + + if ((ctx = tls_client()) == NULL) { + warnx("failed to create tls client"); + return (-1); + } + + if (tls_configure(ctx, tls_config) != 0) { + warnx("tls_configure: %s", tls_error(ctx)); + return (-1); + } + + if (port == NULL) + port = "https"; + + if ((s = http_connect(host, port)) == -1) + return (-1); + + if (tls_connect_socket(ctx, s, host) != 0) { + warnx("tls_connect: %s", tls_error(ctx)); + return (-1); + } + + return (s); +} + +int +https_get(int s, const char *url, const char *fn, int flags) +{ + struct http_headers hdrs; + char hostname[HOST_NAME_MAX+1]; + char *buf; + size_t len; + int fd, res = -1; + + if (gethostname(hostname, sizeof(hostname)) != 0) + hostname[0] = '\0'; + + https_vprintf(ctx, + "GET %s HTTP/1.0\r\n" + "Host: %s\r\n" + "\r\n", + (url) ? url : "/", + hostname); + if ((buf = https_getline(ctx, &len)) == NULL) + goto cleanup; + + fprintf(stderr, "res: %s\n", buf); + if ((res = http_response_code(buf)) == 0) { + warnx("failed to get response code"); + goto cleanup; + } + + memset(&hdrs, 0, sizeof(hdrs)); + if (https_parse_headers(ctx, &hdrs) != 0) + goto cleanup; + + switch (res) { + case 200: + if (strcmp(fn, "-") == 0) + fd = STDOUT_FILENO; + else if ((fd = open(fn, flags, 0666)) == -1) + err(1, "open %s", fn); + + https_200(ctx, fd); + if (fd != STDOUT_FILENO) + close(fd); + break; + } + +cleanup: + tls_close(ctx); + tls_free(ctx); + return (res); +} + +void +https_200(struct tls *ctx, int out) +{ + size_t r, wlen; + ssize_t i; + char *cp; + static char *buf; + + if (buf == NULL) { + buf = malloc(TMPBUF_LEN); /* allocate once */ + if (buf == NULL) + err(1, "malloc"); + } + + /* XXX Content-Length or till EOF? */ + while (tls_read(ctx, buf, TMPBUF_LEN, &r) == 0 && r > 0) { + for (cp = buf, wlen = r; wlen > 0; wlen -= i, cp += i) { + if ((i = write(out, cp, wlen)) == -1) { + warn("https_200: write"); + break; + } else if (i == 0) + break; + } + } +} + +int +https_vprintf(struct tls *tls, const char *fmt, ...) +{ + va_list ap; + char *string; + size_t nw; + int ret; + + va_start(ap, fmt); + if ((ret = vasprintf(&string, fmt, ap)) == -1) + return ret; + + va_end(ap); + ret = tls_write(tls, string, ret, &nw); + free(string); + return ret; +} + +char * +https_getline(struct tls *tls, size_t *lenp) +{ + size_t i, len, nr; + char *buf, *q, c; + int ret; + + len = 128; + if ((buf = malloc(len)) == NULL) + errx(1, "Can't allocate memory for transfer buffer"); + + for (i = 0; ; i++) { + if (i >= len - 1) { + if ((q = reallocarray(buf, len, 2)) == NULL) + errx(1, "Can't expand transfer buffer"); + + buf = q; + len *= 2; + } +again: + ret = tls_read(tls, &c, 1, &nr); + if (ret == TLS_READ_AGAIN) + goto again; + + if (ret != 0) + errx(1, "SSL read error: %u", ret); + + buf[i] = c; + if (c == '\n') + break; + } + + buf[i] = '\0'; + if (buf[i - 1] == '\r') + buf[--i] = '\0'; + + *lenp = i; + return (buf); +} + +int +https_parse_headers(struct tls *ctx, struct http_headers *hdrs) +{ + char *buf; + size_t len; + + while ((buf = https_getline(ctx, &len))) { + if (len == 0) + break; /* end of headers */ + + if (header_insert(hdrs, buf) != 0) + return (-1); + } + + return (0); +} Index: main.c =================================================================== RCS file: main.c diff -N main.c --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ main.c 21 May 2015 18:23:37 -0000 @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2015 Sunil Nimmagadda <su...@nimmagadda.net> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#include <err.h> +#include <fcntl.h> +#include <libgen.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#include "ftp.h" + +#define FTP 1 +#define HTTP 2 +#define HTTPS 3 +#define UNDEF 4 + +struct proto *lookup(int); +int proto_get(const char *); +void usage(void); + +int +main(int argc, char *argv[]) +{ + struct proto *proto; + const char *fn, *output = NULL, *port = NULL; + int ch, code, flags, i, p, r = 1, s = -1; + char host[HOST_NAME_MAX+1]; + + while ((ch = getopt(argc, argv, "o:P:")) != -1) { + switch (ch) { + case 'o': + output = optarg; + break; + case 'P': + port = optarg; + break; + default: + usage(); + } + } + + argc -= optind; + argv += optind; + if (argc == 0) + usage(); + + for (i = 0; i < argc; i++) { + if ((p = proto_get(argv[i])) == UNDEF) { + warnx("skipping unknown protocol: %s\n", argv[i]); + continue; + } + + fn = (output) ? output : basename(argv[i]); + flags = O_CREAT | O_WRONLY; + if (output) + flags |= O_APPEND; + + if (host_extract(argv[i], host, sizeof(host)) != 0) { + warnx("couldn't extract hostname"); + return (-1); + } + + proto = lookup(p); + if ((s = proto->connect(host, port)) == -1) { + warnx("failed connecting to %s\n", host); + return (-1); + } + + if ((code = proto->get(s, argv[i], fn, flags)) != 200) { + warnx("%s failed with code: %d\n", argv[i], code); + close(s); + return (-1); + } + + close(s); + } + + return (r); +} + +struct proto * +lookup(int p) +{ + extern struct proto proto_http; +#ifndef SMALL + extern struct proto proto_https; + extern struct proto proto_ftp; +#endif + + switch (p) { + case HTTP: + return (&proto_http); +#ifndef SMALL + case HTTPS: + return (&proto_https); + case FTP: + return (&proto_ftp); + break; +#endif + } + + errx(1, "invalid proto %d\n", p); + return (NULL); +} + +int +proto_get(const char *url) +{ + if (strncasecmp(url, "http://", 7) == 0) + return (HTTP); +#ifndef SMALL + else if (strncasecmp(url, "https://", 8) == 0) + return (HTTPS); + else if (strncasecmp(url, "ftp://", 6) == 0) + return (FTP); +#endif + else + return (UNDEF); +} + +int +host_extract(char *url, char *host, size_t host_sz) +{ + char *curl, *start, *end; + + if ((curl = strdup(url)) == NULL) + err(1, "host_extract: strdup"); + + if ((start = strstr(curl, "//")) == NULL) + return (-1); + + start += 2; + end = strchr(start, '/'); + if (end) + *end = '\0'; + + if (strlcpy(host, start, host_sz) >= host_sz) { + warnx("host overflow"); + free(curl); + return (-1); + } + + free(curl); + return (0); +} + +int +header_insert(struct http_headers *hdrs, const char *buf) +{ + const char *errstr; + size_t sz; + + if (strncasecmp(buf, "Content-Length:", 15) == 0) { + buf = strchr(buf, ' '); + hdrs->c_len = strtonum(buf, 0, LLONG_MAX, &errstr); + if (errstr) { + warn("insert_header: strtonum"); + return (-1); + } + } + + if (strncasecmp(buf, "Location:", 8) == 0) { + buf = strchr(buf, ' '); + sz = strlcpy(hdrs->location, buf, sizeof(hdrs->location)); + if (sz >= sizeof(hdrs->location)) { + warnx("header Location overflow"); + return (-1); + } + } + + return (0); +} + +void +usage(void) +{ + fprintf(stderr, "usage: %s [-o output] [-P port] url ...\n", + getprogname()); + exit(0); +}