From: claudio Date: Thu, 4 Mar 2021 13:01:41 +0000 (+0000) Subject: Implement a https client as a sub-process for rpki-client. This code will X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=1ef5b48a83d785691f8950d5d9b0a0f3440f1bc6;p=openbsd Implement a https client as a sub-process for rpki-client. This code will be used to fetch TA certs and later on for RRDP. Kind of unreached for now since the default TAL files don't include https URI. The http client is fully asynchronous and can handle multiple downloads at the same time. This code was based on the http client in ftp(1). OK tb@, job@ --- diff --git a/usr.sbin/rpki-client/Makefile b/usr.sbin/rpki-client/Makefile index 2ed7edab7ed..16713b946ca 100644 --- a/usr.sbin/rpki-client/Makefile +++ b/usr.sbin/rpki-client/Makefile @@ -1,13 +1,13 @@ -# $OpenBSD: Makefile,v 1.18 2021/02/04 08:10:24 claudio Exp $ +# $OpenBSD: Makefile,v 1.19 2021/03/04 13:01:41 claudio Exp $ PROG= rpki-client -SRCS= as.c cert.c cms.c crl.c gbr.c io.c ip.c log.c main.c mft.c mkdir.c \ - output.c output-bgpd.c output-bird.c output-csv.c output-json.c \ - parser.c roa.c rsync.c tal.c validate.c x509.c +SRCS= as.c cert.c cms.c crl.c gbr.c http.c io.c ip.c log.c main.c mft.c \ + mkdir.c output.c output-bgpd.c output-bird.c output-csv.c \ + output-json.c parser.c roa.c rsync.c tal.c validate.c x509.c MAN= rpki-client.8 -LDADD+= -lcrypto -lutil -DPADD+= ${LIBCRYPTO} ${LIBUTIL} +LDADD+= -ltls -lssl -lcrypto -lutil +DPADD+= ${LIBTLS} ${LIBSSL} ${LIBCRYPTO} ${LIBUTIL} CFLAGS+= -Wall -I${.CURDIR} CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes diff --git a/usr.sbin/rpki-client/extern.h b/usr.sbin/rpki-client/extern.h index 17face16f84..0e5e6c80c57 100644 --- a/usr.sbin/rpki-client/extern.h +++ b/usr.sbin/rpki-client/extern.h @@ -1,4 +1,4 @@ -/* $OpenBSD: extern.h,v 1.49 2021/03/02 09:23:59 claudio Exp $ */ +/* $OpenBSD: extern.h,v 1.50 2021/03/04 13:01:41 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -399,6 +399,10 @@ void proc_parser(int) __attribute__((noreturn)); char *rsync_base_uri(const char *); void proc_rsync(char *, char *, int) __attribute__((noreturn)); +/* Http-specific. */ + +void proc_http(char *, int); + /* Logging (though really used for OpenSSL errors). */ void cryptowarnx(const char *, ...) @@ -417,6 +421,7 @@ void io_str_buffer(struct ibuf *, const char *); void io_simple_read(int, void *, size_t); void io_buf_read_alloc(int, void **, size_t *); void io_str_read(int, char **); +int io_recvfd(int, void *, size_t); /* X509 helpers. */ diff --git a/usr.sbin/rpki-client/http.c b/usr.sbin/rpki-client/http.c new file mode 100644 index 00000000000..cb2d9dc344d --- /dev/null +++ b/usr.sbin/rpki-client/http.c @@ -0,0 +1,1230 @@ +/* $OpenBSD: http.c,v 1.1 2021/03/04 13:01:41 claudio Exp $ */ +/* + * Copyright (c) 2020 Nils Fisher + * Copyright (c) 2020 Claudio Jeker + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/*- + * Copyright (c) 1997 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jason Thorpe and Luke Mewburn. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "extern.h" + +#define HTTP_USER_AGENT "OpenBSD rpki-client" +#define HTTP_BUF_SIZE (32 * 1024) +#define MAX_CONNECTIONS 12 + +#define WANT_POLLIN 1 +#define WANT_POLLOUT 2 + +enum http_state { + STATE_FREE, + STATE_INIT, + STATE_CONNECT, + STATE_TLSCONNECT, + STATE_REQUEST, + STATE_RESPONSE_STATUS, + STATE_RESPONSE_HEADER, + STATE_RESPONSE_DATA, + STATE_RESPONSE_CHUNKED, + STATE_WRITE_DATA, + STATE_DONE, +}; + +struct http_proxy { + char *proxyhost; + char *proxyuser; + char *proxypw; +}; + +struct http_connection { + char *url; + char *host; + char *port; + const char *path; /* points into url */ + char *modified_since; + char *last_modified; + struct addrinfo *res0; + struct addrinfo *res; + struct tls *tls; + char *buf; + size_t bufsz; + size_t bufpos; + size_t id; + size_t chunksz; + off_t filesize; + off_t bytes; + int status; + int redirect_loop; + int fd; + int outfd; + short events; + short chunked; + enum http_state state; +}; + +struct msgbuf msgq; +struct sockaddr_storage http_bindaddr; +struct tls_config *tls_config; +uint8_t *tls_ca_mem; +size_t tls_ca_size; + +char *resp_buf[MAX_CONNECTIONS]; +size_t resp_bsz[MAX_CONNECTIONS]; +size_t resp_idx; + +/* + * Return a string that can be used in error message to identify the + * connection. + */ +static const char * +http_info(const char *url) +{ + static char buf[64]; + + if (strnvis(buf, url, sizeof buf, VIS_SAFE) >= (int)sizeof buf) { + /* overflow, add indicator */ + memcpy(buf + sizeof buf - 4, "...", 4); + } + + return buf; +} + +/* + * Determine whether the character needs encoding, per RFC1738: + * - No corresponding graphic US-ASCII. + * - Unsafe characters. + */ +static int +unsafe_char(const char *c0) +{ + const char *unsafe_chars = " <>\"#{}|\\^~[]`"; + const unsigned char *c = (const unsigned char *)c0; + + /* + * No corresponding graphic US-ASCII. + * Control characters and octets not used in US-ASCII. + */ + return (iscntrl(*c) || !isascii(*c) || + + /* + * Unsafe characters. + * '%' is also unsafe, if is not followed by two + * hexadecimal digits. + */ + strchr(unsafe_chars, *c) != NULL || + (*c == '%' && (!isxdigit(*++c) || !isxdigit(*++c)))); +} + +/* + * Encode given URL, per RFC1738. + * Allocate and return string to the caller. + */ +static char * +url_encode(const char *path) +{ + size_t i, length, new_length; + char *epath, *epathp; + + length = new_length = strlen(path); + + /* + * First pass: + * Count unsafe characters, and determine length of the + * final URL. + */ + for (i = 0; i < length; i++) + if (unsafe_char(path + i)) + new_length += 2; + + epath = epathp = malloc(new_length + 1); /* One more for '\0'. */ + if (epath == NULL) + err(1, NULL); + + /* + * Second pass: + * Encode, and copy final URL. + */ + for (i = 0; i < length; i++) + if (unsafe_char(path + i)) { + snprintf(epathp, 4, "%%" "%02x", + (unsigned char)path[i]); + epathp += 3; + } else + *(epathp++) = path[i]; + + *epathp = '\0'; + return (epath); +} + +static void +http_setup(void) +{ + tls_config = tls_config_new(); + if (tls_config == NULL) + errx(1, "tls config failed"); +#if 0 + /* TODO Should we allow extra protos and ciphers? */ + if (tls_config_set_protocols(tls_config, TLS_PROTOCOLS_ALL) == -1) + errx(1, "tls set protocols failed: %s", + tls_config_error(tls_config)); + if (tls_config_set_ciphers(tls_config, "legacy") == -1) + errx(1, "tls set ciphers failed: %s", + tls_config_error(tls_config)); +#endif + + /* load cert file from disk now */ + tls_ca_mem = tls_load_file(tls_default_ca_cert_file(), + &tls_ca_size, NULL); + if (tls_ca_mem == NULL) + err(1, "tls_load_file"); + tls_config_set_ca_mem(tls_config, tls_ca_mem, tls_ca_size); + + /* TODO initalize proxy settings */ + +} + +static int +http_resolv(struct http_connection *conn, const char *host, const char *port) +{ + struct addrinfo hints; + int error; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = PF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + error = getaddrinfo(host, port, &hints, &conn->res0); + /* + * If the services file is corrupt/missing, fall back + * on our hard-coded defines. + */ + if (error == EAI_SERVICE) + error = getaddrinfo(host, "443", &hints, &conn->res0); + if (error != 0) { + warnx("%s: %s", host, gai_strerror(error)); + return -1; + } + + return 0; +} + +static void +http_done(struct http_connection *conn, int ok) +{ + struct ibuf *b; + + conn->state = STATE_DONE; + + if ((b = ibuf_dynamic(64, UINT_MAX)) == NULL) + err(1, NULL); + io_simple_buffer(b, &conn->id, sizeof(conn->id)); + io_simple_buffer(b, &ok, sizeof(ok)); +#if 0 /* TODO: cache last_modified */ + io_str_buffer(b, conn->last_modified); +#endif + ibuf_close(&msgq, b); +} + +static void +http_fail(size_t id) +{ + struct ibuf *b; + int ok = 0; + + if ((b = ibuf_dynamic(8, UINT_MAX)) == NULL) + err(1, NULL); + io_simple_buffer(b, &id, sizeof(id)); + io_simple_buffer(b, &ok, sizeof(ok)); + ibuf_close(&msgq, b); +} + +static void +http_free(struct http_connection *conn) +{ + if (conn->state != STATE_DONE) + http_fail(conn->id); + + free(conn->url); + free(conn->host); + free(conn->port); + /* no need to free conn->path it points into conn->url */ + free(conn->modified_since); + free(conn->last_modified); + free(conn->buf); + + if (conn->res0 != NULL) + freeaddrinfo(conn->res0); + + tls_free(conn->tls); + + if (conn->fd != -1) + close(conn->fd); + close(conn->outfd); + free(conn); +} + +static int +http_close(struct http_connection *conn) +{ + if (conn->tls != NULL) { + switch (tls_close(conn->tls)) { + case TLS_WANT_POLLIN: + return WANT_POLLIN; + case TLS_WANT_POLLOUT: + return WANT_POLLOUT; + case 0: + break; + case -1: + warnx("%s: TLS close: %s", http_info(conn->url), + tls_error(conn->tls)); + return -1; + } + } + + return -1; +} + +static int +http_parse_uri(char *uri, char **ohost, char **oport, char **opath) +{ + char *host, *port = NULL, *path; + char *hosttail; + + if (strncasecmp(uri, "https://", 8) != 0) { + warnx("%s: not using https schema", http_info(uri)); + return -1; + } + host = uri + 8; + if ((path = strchr(host, '/')) == NULL) { + warnx("%s: missing https path", http_info(uri)); + return -1; + } + if (path - uri > INT_MAX - 1) { + warnx("%s: preposterous host length", http_info(uri)); + return -1; + } + if (*host == '[') { + char *scope; + if ((hosttail = memrchr(host, ']', path - host)) != NULL && + (hosttail[1] == '/' || hosttail[1] == ':')) + host++; + if (hosttail[1] == ':') + port = hosttail + 1; + if ((scope = memchr(host, '%', hosttail - host)) != NULL) + hosttail = scope; + } else { + if ((hosttail = memrchr(host, ':', path - host)) != NULL) + port = hosttail + 1; + else + hosttail = path; + } + + if ((host = strndup(host, hosttail - host)) == NULL) + err(1, "strndup"); + if (port != NULL) { + if ((port = strndup(port, path - port)) == NULL) + err(1, "strndup"); + } else { + if ((port = strdup("443")) == NULL) + err(1, "strdup"); + } + /* do not include the initial / in path */ + path++; + + *ohost = host; + *oport = port; + *opath = path; + + return 0; +} + + +static struct http_connection * +http_new(size_t id, char *uri, char *modified_since, int outfd) +{ + struct http_connection *conn; + char *host, *port, *path; + + if (http_parse_uri(uri, &host, &port, &path) == -1) { + free(uri); + free(modified_since); + return NULL; + } + + if ((conn = calloc(1, sizeof(*conn))) == NULL) + err(1, NULL); + + conn->id = id; + conn->fd = -1; + conn->outfd = outfd; + conn->host = host; + conn->port = port; + conn->path = path; + conn->url = uri; + conn->modified_since = modified_since; + conn->state = STATE_INIT; + + /* TODO proxy support (overload of host and port) */ + + if (http_resolv(conn, host, port) == -1) { + http_free(conn); + return NULL; + } + + return conn; +} + +static int +http_redirect(struct http_connection *conn, char *uri) +{ + char *host, *port, *path; + + warnx("redirect to %s", http_info(uri)); + + if (http_parse_uri(uri, &host, &port, &path) == -1) { + free(uri); + return -1; + } + + free(conn->url); + conn->url = uri; + free(conn->host); + conn->host = host; + free(conn->port); + conn->port = port; + conn->path = path; + /* keep modified_since since that is part of the request */ + free(conn->last_modified); + conn->last_modified = NULL; + free(conn->buf); + conn->buf = NULL; + conn->bufpos = 0; + conn->bufsz = 0; + tls_close(conn->tls); + tls_free(conn->tls); + conn->tls = NULL; + close(conn->fd); + conn->state = STATE_INIT; + + /* TODO proxy support (overload of host and port) */ + + if (http_resolv(conn, host, port) == -1) + return -1; + + return 0; +} + +static int +http_connect(struct http_connection *conn) +{ + char pbuf[NI_MAXSERV], hbuf[NI_MAXHOST]; + char *cause = "unknown"; + + if (conn->fd != -1) { + close(conn->fd); + conn->fd = -1; + } + + /* start the loop below with first or next address */ + if (conn->res == NULL) + conn->res = conn->res0; + else + conn->res = conn->res->ai_next; + for (; conn->res != NULL; conn->res = conn->res->ai_next) { + struct addrinfo *res = conn->res; + int fd, error, save_errno; + + if (getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, + sizeof(hbuf), pbuf, sizeof(pbuf), + NI_NUMERICHOST | NI_NUMERICSERV) != 0) + strlcpy(hbuf, "(unknown)", sizeof(hbuf)); + + fd = socket(res->ai_family, + res->ai_socktype | SOCK_NONBLOCK, res->ai_protocol); + if (fd == -1) { + cause = "socket"; + continue; + } + conn->fd = fd; + + if (http_bindaddr.ss_family == res->ai_family) { + if (bind(conn->fd, (struct sockaddr *)&http_bindaddr, + res->ai_addrlen) == -1) + warn("%s: bind", http_info(conn->url)); + } + + error = connect(conn->fd, res->ai_addr, res->ai_addrlen); + if (error == -1) { + if (errno == EINPROGRESS) { + /* waiting for connect to finish. */ + return WANT_POLLOUT; + } else { + save_errno = errno; + close(conn->fd); + conn->fd = -1; + errno = save_errno; + cause = "connect"; + continue; + } + } + + /* TODO proxy connect */ +#if 0 + if (proxyenv) + proxy_connect(conn->fd, sslhost, proxy_credentials); */ +#endif + } + freeaddrinfo(conn->res0); + conn->res0 = NULL; + if (conn->fd == -1) { + warn("%s: %s", http_info(conn->url), cause); + return -1; + } + return 0; +} + +static int +http_finish_connect(struct http_connection *conn) +{ + int error = 0; + socklen_t len; + + len = sizeof(error); + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &error, &len) == -1) { + warn("%s: getsockopt SO_ERROR", http_info(conn->url)); + /* connection will be closed by http_connect() */ + return -1; + } + if (error != 0) { + errno = error; + warn("%s: connect", http_info(conn->url)); + return -1; + } + + return 0; +} + +static int +http_tls_handshake(struct http_connection *conn) +{ + switch (tls_handshake(conn->tls)) { + case 0: + return 0; + case TLS_WANT_POLLIN: + return WANT_POLLIN; + case TLS_WANT_POLLOUT: + return WANT_POLLOUT; + } + warnx("%s: TLS handshake: %s", http_info(conn->url), + tls_error(conn->tls)); + return -1; +} + +static int +http_tls_connect(struct http_connection *conn) +{ + if ((conn->tls = tls_client()) == NULL) { + warn("tls_client"); + return -1; + } + if (tls_configure(conn->tls, tls_config) == -1) { + warnx("%s: TLS configuration: %s\n", http_info(conn->url), + tls_error(conn->tls)); + return -1; + } + if (tls_connect_socket(conn->tls, conn->fd, conn->host) == -1) { + warnx("%s: TLS connect: %s\n", http_info(conn->url), + tls_error(conn->tls)); + return -1; + } + return http_tls_handshake(conn); +} + +static int +http_request(struct http_connection *conn) +{ + char *host, *epath, *modified_since; + int with_port = 0; + + /* TODO adjust request for HTTP proxy setups */ + + /* + * Send port number only if it's specified and does not equal + * the default. Some broken HTTP servers get confused if you explicitly + * send them the port number. + */ + if (conn->port && strcmp(conn->port, "443") != 0) + with_port = 1; + + /* Construct the Host header from host and port info */ + if (strchr(conn->host, ':')) { + if (asprintf(&host, "[%s]%s%s", conn->host, + with_port ? ":" : "", with_port ? conn->port : "") == -1) + err(1, NULL); + + } else { + if (asprintf(&host, "%s%s%s", conn->host, + with_port ? ":" : "", with_port ? conn->port : "") == -1) + err(1, NULL); + } + + /* + * Construct and send the request. Proxy requests don't want leading /. + */ + epath = url_encode(conn->path); + + modified_since = NULL; + if (conn->modified_since) { + if (asprintf(&modified_since, "If-Modified-Since: %s\r\n", + conn->modified_since) == -1) + err(1, NULL); + } + + free(conn->buf); + conn->bufpos = 0; + if ((conn->bufsz = asprintf(&conn->buf, + "GET /%s HTTP/1.1\r\n" + "Connection: close\r\n" + "User-Agent: " HTTP_USER_AGENT "\r\n" + "Host: %s\r\n%s\r\n", + epath, host, + modified_since ? modified_since : "")) == -1) + err(1, NULL); + + free(epath); + free(host); + free(modified_since); + + return WANT_POLLOUT; +} + +static int +http_write(struct http_connection *conn) +{ + ssize_t s; + + s = tls_write(conn->tls, conn->buf + conn->bufpos, + conn->bufsz - conn->bufpos); + if (s == -1) { + warnx("%s: TLS write: %s", http_info(conn->url), + tls_error(conn->tls)); + return -1; + } else if (s == TLS_WANT_POLLIN) { + return WANT_POLLIN; + } else if (s == TLS_WANT_POLLOUT) { + return WANT_POLLOUT; + } + + conn->bufpos += s; + if (conn->bufpos == conn->bufsz) + return 0; + return WANT_POLLOUT; +} + +static int +http_parse_status(struct http_connection *conn, char *buf) +{ + const char *errstr; + char *cp, ststr[4]; + char gerror[200]; + int status; + + cp = strchr(buf, ' '); + if (cp == NULL) { + warnx("Improper response from %s", http_info(conn->url)); + return -1; + } else + cp++; + + strlcpy(ststr, cp, sizeof(ststr)); + status = strtonum(ststr, 200, 599, &errstr); + if (errstr != NULL) { + strnvis(gerror, cp, sizeof gerror, VIS_SAFE); + warnx("Error retrieving %s: %s", http_info(conn->url), gerror); + return -1; + } + + switch (status) { + case 301: + case 302: + case 303: + case 307: + if (conn->redirect_loop++ > 10) { + warnx("%s: Too many redirections requested", + http_info(conn->url)); + return -1; + } + /* FALLTHROUGH */ + case 200: + case 206: + case 304: + conn->status = status; + break; + default: + strnvis(gerror, cp, sizeof gerror, VIS_SAFE); + warnx("Error retrieving %s: %s", http_info(conn->url), gerror); + break; + } + + return 0; +} + +static inline int +http_isredirect(struct http_connection *conn) +{ + if ((conn->status >= 301 && conn->status <= 303) || + conn->status == 307) + return 1; + return 0; +} + +static int +http_parse_header(struct http_connection *conn, char *buf) +{ +#define CONTENTLEN "Content-Length: " +#define LOCATION "Location: " +#define TRANSFER_ENCODING "Transfer-Encoding: " +#define LAST_MODIFIED "Last-Modified: " + const char *errstr; + char *cp, *redirurl; + char *locbase, *loctail; + + cp = buf; + /* empty line, end of header */ + if (*cp == '\0') + return 0; + else if (strncasecmp(cp, CONTENTLEN, sizeof(CONTENTLEN) - 1) == 0) { + size_t s; + cp += sizeof(CONTENTLEN) - 1; + if ((s = strcspn(cp, " \t")) != 0) + *(cp+s) = 0; + conn->filesize = strtonum(cp, 0, LLONG_MAX, &errstr); + if (errstr != NULL) { + warnx("Improper response from %s", + http_info(conn->url)); + return -1; + } + } else if (http_isredirect(conn) && + strncasecmp(cp, LOCATION, sizeof(LOCATION) - 1) == 0) { + cp += sizeof(LOCATION) - 1; + /* + * If there is a colon before the first slash, this URI + * is not relative. RFC 3986 4.2 + */ + if (cp[strcspn(cp, ":/")] != ':') { + /* XXX doesn't handle protocol-relative URIs */ + if (*cp == '/') { + locbase = NULL; + cp++; + } else { + locbase = strdup(conn->path); + if (locbase == NULL) + err(1, ""); + loctail = strchr(locbase, '#'); + if (loctail != NULL) + *loctail = '\0'; + loctail = strchr(locbase, '?'); + if (loctail != NULL) + *loctail = '\0'; + loctail = strrchr(locbase, '/'); + if (loctail == NULL) { + free(locbase); + locbase = NULL; + } else + loctail[1] = '\0'; + } + /* Construct URL from relative redirect */ + if (asprintf(&redirurl, "%.*s/%s%s", + (int)(conn->path - conn->url), conn->url, + locbase ? locbase : "", + cp) == -1) + err(1, "Cannot build redirect URL"); + free(locbase); + } else if ((redirurl = strdup(cp)) == NULL) + err(1, "Cannot build redirect URL"); + loctail = strchr(redirurl, '#'); + if (loctail != NULL) + *loctail = '\0'; + return http_redirect(conn, redirurl); + } else if (strncasecmp(cp, TRANSFER_ENCODING, + sizeof(TRANSFER_ENCODING) - 1) == 0) { + cp += sizeof(TRANSFER_ENCODING) - 1; + cp[strcspn(cp, " \t")] = '\0'; + if (strcasecmp(cp, "chunked") == 0) + conn->chunked = 1; + } else if (strncasecmp(cp, LAST_MODIFIED, + sizeof(LAST_MODIFIED) - 1) == 0) { + cp += sizeof(LAST_MODIFIED) - 1; + conn->last_modified = strdup(cp); + } + + return 1; +} + +static char * +http_get_line(struct http_connection *conn) +{ + char *end, *line; + size_t len; + + end = memchr(conn->buf, '\n', conn->bufpos); + if (end == NULL) + return NULL; + + len = end - conn->buf; + while (len > 0 && conn->buf[len - 1] == '\r') + --len; + if ((line = strndup(conn->buf, len)) == NULL) + err(1, "%s", __func__); + + /* consume line including \n */ + end++; + conn->bufpos -= end - conn->buf; + memmove(conn->buf, end, conn->bufpos); + + return line; +} + +static int +http_parse_chunked(struct http_connection *conn, char *buf) +{ + char *header = buf; + char *end; + int chunksize; + + /* ignore empty lines, used between chunk and next header */ + if (*header == '\0') + return 1; + + /* strip CRLF and any optional chunk extension */ + header[strcspn(header, ";\r\n")] = '\0'; + errno = 0; + chunksize = strtoul(header, &end, 16); + if (errno != 0 || header[0] == '\0' || *end != '\0' || + chunksize > INT_MAX) { + warnx("%s: Invalid chunk size", http_info(conn->url)); + return -1; + } + conn->chunksz = chunksize; + + if (conn->chunksz == 0) { + http_done(conn, 1); + return 0; + } + + return 1; +} + +static int +http_read(struct http_connection *conn) +{ + ssize_t s; + char *buf; + + s = tls_read(conn->tls, conn->buf + conn->bufpos, + conn->bufsz - conn->bufpos); + if (s == -1) { + warn("%s: TLS read: %s", http_info(conn->url), + tls_error(conn->tls)); + return -1; + } else if (s == TLS_WANT_POLLIN) { + return WANT_POLLIN; + } else if (s == TLS_WANT_POLLOUT) { + return WANT_POLLOUT; + } + + if (s == 0 && conn->bufpos == 0) { + warnx("%s: short read, connection closed", + http_info(conn->url)); + return -1; + } + + conn->bufpos += s; + + switch (conn->state) { + case STATE_RESPONSE_STATUS: + buf = http_get_line(conn); + if (buf == NULL) + return WANT_POLLIN; + if (http_parse_status(conn, buf) == -1) { + free(buf); + return -1; + } + free(buf); + conn->state = STATE_RESPONSE_HEADER; + /* FALLTHROUGH */ + case STATE_RESPONSE_HEADER: + while ((buf = http_get_line(conn)) != NULL) { + int rv; + + rv = http_parse_header(conn, buf); + free(buf); + if (rv != 1) + return rv; + } + return WANT_POLLIN; + case STATE_RESPONSE_DATA: + if (conn->bufpos == conn->bufsz || + conn->filesize - conn->bytes <= (off_t)conn->bufpos) + return 0; + return WANT_POLLIN; + case STATE_RESPONSE_CHUNKED: + while (conn->chunksz == 0) { + buf = http_get_line(conn); + if (buf == NULL) + return WANT_POLLIN; + switch (http_parse_chunked(conn, buf)) { + case -1: + free(buf); + return -1; + case 0: + free(buf); + return 0; + } + free(buf); + } + + if (conn->bufpos == conn->bufsz || + conn->chunksz <= conn->bufpos) + return 0; + return WANT_POLLIN; + default: + errx(1, "unexpected http state"); + } +} + +static int +data_write(struct http_connection *conn) +{ + ssize_t s; + size_t bsz = conn->bufpos; + + if (conn->filesize - conn->bytes < (off_t)bsz) + bsz = conn->filesize - conn->bytes; + + s = write(conn->outfd, conn->buf, bsz); + if (s == -1) { + warn("%s: Data write", http_info(conn->url)); + return -1; + } + + conn->bufpos -= s; + conn->bytes += s; + memmove(conn->buf, conn->buf + s, conn->bufpos); + + if (conn->bytes == conn->filesize) { + http_done(conn, 1); + return 0; + } + + if (conn->bufpos == 0) + return 0; + + return WANT_POLLOUT; +} + +static int +chunk_write(struct http_connection *conn) +{ + ssize_t s; + size_t bsz = conn->bufpos; + + if (bsz > conn->chunksz) + bsz = conn->chunksz; + + s = write(conn->outfd, conn->buf, bsz); + if (s == -1) { + warn("%s: Chunk write", http_info(conn->url)); + return -1; + } + + conn->bufpos -= s; + conn->chunksz -= s; + conn->bytes += s; + memmove(conn->buf, conn->buf + s, conn->bufpos); + + if (conn->bufpos == 0 || conn->chunksz == 0) + return 0; + + return WANT_POLLOUT; +} + +static int +http_handle(struct http_connection *conn) +{ + switch (conn->state) { + case STATE_INIT: + return 0; + case STATE_CONNECT: + if (http_finish_connect(conn) == -1) + /* something went wrong, try other host */ + return http_connect(conn); + return 0; + case STATE_TLSCONNECT: + return http_tls_handshake(conn); + case STATE_REQUEST: + return http_write(conn); + case STATE_RESPONSE_STATUS: + case STATE_RESPONSE_HEADER: + case STATE_RESPONSE_DATA: + case STATE_RESPONSE_CHUNKED: + return http_read(conn); + case STATE_WRITE_DATA: + if (conn->chunked) + return chunk_write(conn); + else + return data_write(conn); + case STATE_DONE: + return http_close(conn); + case STATE_FREE: + errx(1, "bad http state"); + } +} + +static int +http_nextstep(struct http_connection *conn) +{ + switch (conn->state) { + case STATE_INIT: + conn->state = STATE_CONNECT; + return http_connect(conn); + case STATE_CONNECT: + conn->state = STATE_TLSCONNECT; + return http_tls_connect(conn); + case STATE_TLSCONNECT: + conn->state = STATE_REQUEST; + return http_request(conn); + case STATE_REQUEST: + conn->state = STATE_RESPONSE_STATUS; + free(conn->buf); + /* allocate the read buffer */ + if ((conn->buf = malloc(HTTP_BUF_SIZE)) == NULL) + err(1, NULL); + conn->bufpos = 0; + conn->bufsz = HTTP_BUF_SIZE; + return WANT_POLLIN; + case STATE_RESPONSE_STATUS: + conn->state = STATE_RESPONSE_HEADER; + return WANT_POLLIN; + case STATE_RESPONSE_HEADER: + if (conn->status == 200) + conn->state = STATE_RESPONSE_DATA; + else { + http_done(conn, 0); + return http_close(conn); + } + return WANT_POLLIN; + case STATE_RESPONSE_DATA: + conn->state = STATE_WRITE_DATA; + return WANT_POLLOUT; + case STATE_RESPONSE_CHUNKED: + conn->state = STATE_WRITE_DATA; + return WANT_POLLOUT; + case STATE_WRITE_DATA: + if (conn->chunked) + conn->state = STATE_RESPONSE_CHUNKED; + else + conn->state = STATE_RESPONSE_DATA; + return WANT_POLLIN; + case STATE_DONE: + return http_close(conn); + case STATE_FREE: + errx(1, "bad http state"); + } +} + +static int +http_do(struct http_connection *conn) +{ + switch (http_handle(conn)) { + case -1: + /* connection failure */ + http_free(conn); + return -1; + case 0: + switch (http_nextstep(conn)) { + case WANT_POLLIN: + conn->events = POLLIN; + break; + case WANT_POLLOUT: + conn->events = POLLOUT; + break; + case -1: + http_free(conn); + return -1; + } + break; + case WANT_POLLIN: + conn->events = POLLIN; + break; + case WANT_POLLOUT: + conn->events = POLLOUT; + break; + } + return 0; +} + +void +proc_http(char *bind_addr, int fd) +{ + struct http_connection *http_conns[MAX_CONNECTIONS]; + struct pollfd pfds[MAX_CONNECTIONS + 1]; + size_t i; + int active_connections; + + if (bind_addr != NULL) { + struct addrinfo hints, *res; + + bzero(&hints, sizeof(hints)); + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_DGRAM; /*dummy*/ + hints.ai_flags = AI_NUMERICHOST; + if (getaddrinfo(bind_addr, NULL, &hints, &res) == 0) { + memcpy(&http_bindaddr, res->ai_addr, res->ai_addrlen); + freeaddrinfo(res); + } + } + http_setup(); + + if (pledge("stdio inet dns recvfd", NULL) == -1) + err(1, "pledge"); + + memset(&http_conns, 0, sizeof(http_conns)); + memset(&pfds, 0, sizeof(pfds)); + pfds[MAX_CONNECTIONS].fd = fd; + + msgbuf_init(&msgq); + msgq.fd = fd; + + for (;;) { + active_connections = 0; + for (i = 0; i < MAX_CONNECTIONS; i++) { + struct http_connection *conn = http_conns[i]; + if (conn == NULL) { + pfds[i].fd = -1; + continue; + } + if (conn->state == STATE_WRITE_DATA) + pfds[i].fd = conn->outfd; + else + pfds[i].fd = conn->fd; + + pfds[i].events = conn->events; + active_connections++; + } + pfds[MAX_CONNECTIONS].events = 0; + if (active_connections < MAX_CONNECTIONS) + pfds[MAX_CONNECTIONS].events |= POLLIN; + if (msgq.queued) + pfds[MAX_CONNECTIONS].events |= POLLOUT; + + if (poll(pfds, sizeof(pfds) / sizeof(pfds[0]), INFTIM) == -1) + err(1, "poll"); + + if (pfds[MAX_CONNECTIONS].revents & POLLHUP) + break; + + if (pfds[MAX_CONNECTIONS].revents & POLLIN) { + struct http_connection *h; + size_t id; + int outfd; + char *uri; + char *mod; + + outfd = io_recvfd(fd, &id, sizeof(id)); + io_str_read(fd, &uri); + io_str_read(fd, &mod); + + h = http_new(id, uri, mod, outfd); + if (h == NULL) { + close(outfd); + http_fail(id); + } else + for (i = 0; i < MAX_CONNECTIONS; i++) { + if (http_conns[i] != NULL) + continue; + http_conns[i] = h; + if (http_do(h) == -1) + http_conns[i] = NULL; + break; + } + } + if (pfds[MAX_CONNECTIONS].revents & POLLOUT) { + switch (msgbuf_write(&msgq)) { + case 0: + errx(1, "write: connection closed"); + case -1: + err(1, "write"); + } + } + for (i = 0; i < MAX_CONNECTIONS; i++) { + struct http_connection *conn = http_conns[i]; + + if (conn == NULL) + continue; + /* event not ready */ + if (!(pfds[i].revents & (conn->events | POLLHUP))) + continue; + + if (http_do(conn) == -1) + http_conns[i] = NULL; + } + } + + exit(0); +} diff --git a/usr.sbin/rpki-client/io.c b/usr.sbin/rpki-client/io.c index cade2aa1128..507a2d6b77b 100644 --- a/usr.sbin/rpki-client/io.c +++ b/usr.sbin/rpki-client/io.c @@ -1,4 +1,4 @@ -/* $OpenBSD: io.c,v 1.12 2021/01/08 08:09:07 claudio Exp $ */ +/* $OpenBSD: io.c,v 1.13 2021/03/04 13:01:41 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -16,9 +16,11 @@ */ #include +#include #include #include +#include #include #include #include @@ -146,3 +148,78 @@ io_str_read(int fd, char **res) err(1, NULL); io_simple_read(fd, *res, sz); } + +/* + * Read data from socket but receive a file descriptor at the same time. + */ +int +io_recvfd(int fd, void *res, size_t sz) +{ + struct iovec iov; + struct msghdr msg; + struct cmsghdr *cmsg; + union { + struct cmsghdr hdr; + char buf[CMSG_SPACE(sizeof(int))]; + } cmsgbuf; + int outfd = -1; + char *b = res; + ssize_t n; + + memset(&msg, 0, sizeof(msg)); + memset(&cmsgbuf, 0, sizeof(cmsgbuf)); + + iov.iov_base = res; + iov.iov_len = sz; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_control = &cmsgbuf.buf; + msg.msg_controllen = sizeof(cmsgbuf.buf); + + while ((n = recvmsg(fd, &msg, 0)) == -1) { + if (errno == EINTR) + continue; + err(1, "recvmsg"); + } + + if (n == 0) + errx(1, "recvmsg: unexpected end of file"); + + for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; + cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if (cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_RIGHTS) { + int i, j, f; + + j = ((char *)cmsg + cmsg->cmsg_len - + (char *)CMSG_DATA(cmsg)) / sizeof(int); + for (i = 0; i < j; i++) { + f = ((int *)CMSG_DATA(cmsg))[i]; + if (i == 0) + outfd = f; + else + close(f); + } + } + } + + b += n; + sz -= n; + while (sz > 0) { + /* short receive */ + n = recv(fd, b, sz, 0); + if (n == -1) { + if (errno == EINTR) + continue; + err(1, "recv"); + } + if (n == 0) + errx(1, "recv: unexpected end of file"); + + b += n; + sz -= n; + } + + return outfd; +} diff --git a/usr.sbin/rpki-client/main.c b/usr.sbin/rpki-client/main.c index 9d5740bcb7a..cc31407ccf3 100644 --- a/usr.sbin/rpki-client/main.c +++ b/usr.sbin/rpki-client/main.c @@ -1,4 +1,4 @@ -/* $OpenBSD: main.c,v 1.108 2021/03/02 09:23:59 claudio Exp $ */ +/* $OpenBSD: main.c,v 1.109 2021/03/04 13:01:41 claudio Exp $ */ /* * Copyright (c) 2019 Kristaps Dzonsons * @@ -51,10 +51,12 @@ /* * An rsync repository. */ +#define REPO_MAX_URI 2 struct repo { char *repouri; /* CA repository base URI */ char *local; /* local path name */ - char *uris[2]; /* URIs to fetch from */ + char *temp; /* temporary file / dir */ + char *uris[REPO_MAX_URI]; /* URIs to fetch from */ size_t id; /* identifier (array index) */ int uriidx; /* which URI is fetched */ int loaded; /* whether loaded or not */ @@ -91,7 +93,7 @@ RB_HEAD(filepath_tree, filepath); RB_PROTOTYPE(filepath_tree, filepath, entry, filepathcmp); static struct filepath_tree fpt = RB_INITIALIZER(&fpt); -static struct msgbuf procq, rsyncq; +static struct msgbuf procq, rsyncq, httpq; static int cachefd, outdirfd; const char *bird_tablename = "ROAS"; @@ -101,6 +103,9 @@ int noop; struct stats stats; +static void repo_fetch(struct repo *); +static char *ta_filename(const struct repo *, int); + /* * Log a message to stderr if and only if "verbose" is non-zero. * This uses the err(3) functionality. @@ -278,6 +283,68 @@ repo_alloc(void) return rp; } +static void +http_ta_fetch(struct repo *rp) +{ + struct ibuf *b; + int filefd; + + rp->temp = ta_filename(rp, 1); + + filefd = mkostemp(rp->temp, O_CLOEXEC); + if (filefd == -1) + err(1, "mkostemp: %s", rp->temp); + if (fchmod(filefd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == -1) + warn("fchmod: %s", rp->temp); + + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) + err(1, NULL); + io_simple_buffer(b, &rp->id, sizeof(rp->id)); + io_str_buffer(b, rp->uris[rp->uriidx]); + /* TODO last modified time */ + io_str_buffer(b, NULL); + /* pass file as fd */ + b->fd = filefd; + ibuf_close(&httpq, b); +} + +static int +http_done(struct repo *rp, int ok) +{ + if (rp->repouri == NULL) { + /* Move downloaded TA file into place, or unlink on failure. */ + if (ok) { + char *file; + + file = ta_filename(rp, 0); + if (renameat(cachefd, rp->temp, cachefd, file) == -1) + warn("rename to %s", file); + } else { + if (unlinkat(cachefd, rp->temp, 0) == -1) + warn("unlink %s", rp->temp); + } + free(rp->temp); + rp->temp = NULL; + } + + if (!ok && rp->uriidx < REPO_MAX_URI - 1 && + rp->uris[rp->uriidx + 1] != NULL) { + logx("%s: load from network failed, retry", rp->local); + + rp->uriidx++; + repo_fetch(rp); + return 0; + } + + if (ok) + logx("%s: loaded from network", rp->local); + else + logx("%s: load from network failed, " + "fallback to cache", rp->local); + + return 1; +} + static void repo_fetch(struct repo *rp) { @@ -293,20 +360,31 @@ repo_fetch(struct repo *rp) /* * Create destination location. - * Build up the tree to this point because GPL rsync(1) - * will not build the destination for us. + * Build up the tree to this point. */ if (mkpath(rp->local) == -1) err(1, "%s", rp->local); - logx("%s: pulling from network", rp->local); - if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) - err(1, NULL); - io_simple_buffer(b, &rp->id, sizeof(rp->id)); - io_str_buffer(b, rp->local); - io_str_buffer(b, rp->uris[0]); - ibuf_close(&rsyncq, b); + logx("%s: pulling from %s", rp->local, rp->uris[rp->uriidx]); + + if (strncasecmp(rp->uris[rp->uriidx], "rsync://", 8) == 0) { + if ((b = ibuf_dynamic(256, UINT_MAX)) == NULL) + err(1, NULL); + io_simple_buffer(b, &rp->id, sizeof(rp->id)); + io_str_buffer(b, rp->local); + io_str_buffer(b, rp->uris[rp->uriidx]); + ibuf_close(&rsyncq, b); + } else { + /* + * Two cases for https. TA files load directly while + * for RRDP XML files are downloaded an parsed to build + * the repo. TA repos have a NULL repouri. + */ + if (rp->repouri == NULL) { + http_ta_fetch(rp); + } + } } /* @@ -333,13 +411,11 @@ ta_lookup(const struct tal *tal) rp = repo_alloc(); rp->local = local; for (i = 0, j = 0; i < tal->urisz && j < 2; i++) { - if (strncasecmp(tal->uri[i], "rsync://", 8) != 0) - continue; /* ignore non rsync URI for now */ if ((rp->uris[j++] = strdup(tal->uri[i])) == NULL) err(1, "strdup"); } if (j == 0) - errx(1, "TAL file has no rsync:// URI"); + errx(1, "TAL %s has no URI", tal->descr); repo_fetch(rp); return rp; @@ -379,6 +455,23 @@ repo_lookup(const char *uri) return rp; } +static char * +ta_filename(const struct repo *repo, int temp) +{ + const char *file; + char *nfile; + + /* does not matter which URI, all end with same filename */ + file = strrchr(repo->uris[0], '/'); + assert(file); + + if (asprintf(&nfile, "%s%s%s", repo->local, file, + temp ? ".XXXXXXXX": "") == -1) + err(1, "asprintf"); + + return nfile; +} + /* * Build local file name base on the URI and the repo info. */ @@ -521,18 +614,13 @@ queue_add_from_tal(struct entityq *q, const struct tal *tal) { char *nfile; const struct repo *repo; - const char *uri; assert(tal->urisz); /* Look up the repository. */ repo = ta_lookup(tal); - uri = strrchr(repo->uris[0], '/'); - assert(uri); - if (asprintf(&nfile, "%s/%s", repo->local, uri + 1) == -1) - err(1, "asprintf"); - + nfile = ta_filename(repo, 0); entityq_add(q, nfile, RTYPE_CER, repo, tal->pkey, tal->pkeysz, tal->descr); } @@ -767,13 +855,13 @@ suicide(int sig __attribute__((unused))) int main(int argc, char *argv[]) { - int rc = 1, c, proc, st, rsync, + int rc = 1, c, st, proc, rsync, http, fl = SOCK_STREAM | SOCK_CLOEXEC; size_t i, outsz = 0, talsz = 0; - pid_t procpid, rsyncpid; + pid_t procpid, rsyncpid, httppid; int fd[2]; struct entityq q; - struct pollfd pfd[2]; + struct pollfd pfd[3]; struct roa **out = NULL; char *rsync_prog = "openrsync"; char *bind_addr = NULL; @@ -801,7 +889,8 @@ main(int argc, char *argv[]) cachedir = RPKI_PATH_BASE_DIR; outputdir = RPKI_PATH_OUT_DIR; - if (pledge("stdio rpath wpath cpath fattr proc exec unveil", NULL) == -1) + if (pledge("stdio rpath wpath cpath inet fattr dns sendfd recvfd " + "proc exec unveil", NULL) == -1) err(1, "pledge"); while ((c = getopt(argc, argv, "b:Bcd:e:jnos:t:T:v")) != -1) @@ -947,15 +1036,48 @@ main(int argc, char *argv[]) } else rsync = -1; - assert(rsync != proc); + /* + * Create a process that will fetch data via https. + * With every request the http process receives a file descriptor + * where the data should be written to. + */ + + if (!noop) { + if (socketpair(AF_UNIX, fl, 0, fd) == -1) + err(1, "socketpair"); + if ((httppid = fork()) == -1) + err(1, "fork"); + + if (httppid == 0) { + close(proc); + close(rsync); + close(fd[1]); - if (pledge("stdio rpath wpath cpath fattr", NULL) == -1) + /* change working directory to the cache directory */ + if (fchdir(cachefd) == -1) + err(1, "fchdir"); + + if (pledge("stdio rpath inet dns recvfd", NULL) == -1) + err(1, "pledge"); + + proc_http(bind_addr, fd[0]); + /* NOTREACHED */ + } + + close(fd[0]); + http = fd[1]; + } else + http = -1; + + if (pledge("stdio rpath wpath cpath fattr sendfd", NULL) == -1) err(1, "pledge"); msgbuf_init(&procq); msgbuf_init(&rsyncq); + msgbuf_init(&httpq); procq.fd = proc; rsyncq.fd = rsync; + httpq.fd = http; /* * The main process drives the top-down scan to leaf ROAs using @@ -965,6 +1087,7 @@ main(int argc, char *argv[]) pfd[0].fd = rsync; pfd[1].fd = proc; + pfd[2].fd = http; /* * Prime the process with our TAL file. @@ -982,18 +1105,23 @@ main(int argc, char *argv[]) pfd[1].events = POLLIN; if (procq.queued) pfd[1].events |= POLLOUT; + pfd[2].events = POLLIN; + if (httpq.queued) + pfd[2].events |= POLLOUT; - if ((c = poll(pfd, 2, INFTIM)) == -1) { + if ((c = poll(pfd, 3, INFTIM)) == -1) { if (errno == EINTR) continue; err(1, "poll"); } if ((pfd[0].revents & (POLLERR|POLLNVAL)) || - (pfd[1].revents & (POLLERR|POLLNVAL))) + (pfd[1].revents & (POLLERR|POLLNVAL)) || + (pfd[2].revents & (POLLERR|POLLNVAL))) errx(1, "poll: bad fd"); if ((pfd[0].revents & POLLHUP) || - (pfd[1].revents & POLLHUP)) + (pfd[1].revents & POLLHUP) || + (pfd[2].revents & POLLHUP)) errx(1, "poll: hangup"); if (pfd[0].revents & POLLOUT) { @@ -1012,9 +1140,18 @@ main(int argc, char *argv[]) err(1, "write"); } } + if (pfd[2].revents & POLLOUT) { + switch (msgbuf_write(&httpq)) { + case 0: + errx(1, "write: connection closed"); + case -1: + err(1, "write"); + } + } + /* - * Check the rsync process. + * Check the rsync and http process. * This means that one of our modules has completed * downloading and we can flush the module requests into * the parser process. @@ -1027,17 +1164,31 @@ main(int argc, char *argv[]) assert(i < rt.reposz); assert(!rt.repos[i].loaded); - rt.repos[i].loaded = 1; if (ok) logx("%s: loaded from network", rt.repos[i].local); else logx("%s: load from network failed, " "fallback to cache", rt.repos[i].local); + rt.repos[i].loaded = 1; stats.repos++; entityq_flush(&q, &rt.repos[i]); } + if ((pfd[2].revents & POLLIN)) { + int ok; + io_simple_read(http, &i, sizeof(size_t)); + io_simple_read(http, &ok, sizeof(ok)); + assert(i < rt.reposz); + + assert(!rt.repos[i].loaded); + if (http_done(&rt.repos[i], ok)) { + rt.repos[i].loaded = 1; + stats.repos++; + entityq_flush(&q, &rt.repos[i]); + } + } + /* * The parser has finished something for us. * Dequeue these one by one. @@ -1066,6 +1217,7 @@ main(int argc, char *argv[]) close(proc); close(rsync); + close(http); if (waitpid(procpid, &st, 0) == -1) err(1, "waitpid"); @@ -1080,6 +1232,13 @@ main(int argc, char *argv[]) warnx("rsync process exited abnormally"); rc = 1; } + + if (waitpid(httppid, &st, 0) == -1) + err(1, "waitpid"); + if (!WIFEXITED(st) || WEXITSTATUS(st) != 0) { + warnx("http process exited abnormally"); + rc = 1; + } } stats.del_files = repo_cleanup(); @@ -1120,6 +1279,7 @@ main(int argc, char *argv[]) for (i = 0; i < rt.reposz; i++) { free(rt.repos[i].repouri); free(rt.repos[i].local); + free(rt.repos[i].temp); free(rt.repos[i].uris[0]); free(rt.repos[i].uris[1]); }