path: root/http.c

/*
 * Copyright (c) 2015 Sunil Nimmagadda <sunil@openbsd.org>
 * Copyright (c) 2012 - 2015 Reyk Floeter <reyk@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <err.h>
#include <fcntl.h>
#include <libgen.h>
#include <limits.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#ifndef NOSSL
#include <tls.h>
#endif /* NOSSL */

#include "ftp.h"
#include "xmalloc.h"

#define MAX_REDIRECTS		10

#ifndef NOSSL
#define MINBUF			128

static struct tls_config	*tls_config;
static struct tls		*ctx;
static int			 tls_session_fd = -1;
static char * const		 tls_verify_opts[] = {
#define HTTP_TLS_CAFILE		0
	"cafile",
#define HTTP_TLS_CAPATH		1
	"capath",
#define HTTP_TLS_CIPHERS	2
	"ciphers",
#define HTTP_TLS_DONTVERIFY	3
	"dont",
#define HTTP_TLS_VERIFYDEPTH	4
	"depth",
#define HTTP_TLS_MUSTSTAPLE	5
	"muststaple",
#define HTTP_TLS_NOVERIFYTIME	6
	"noverifytime",
#define HTTP_TLS_SESSION	7
	"session",
#define HTTP_TLS_DOVERIFY	8
	"do",
	NULL
};
#endif /* NOSSL */

/*
 * HTTP status codes based on IANA assignments (2014-06-11 version):
 * https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
 * plus legacy (306) and non-standard (420).
 */
static struct http_status {
	int		 code;
	const char	*name;
} http_status[] = {
	{ 100,	"Continue" },
	{ 101,	"Switching Protocols" },
	{ 102,	"Processing" },
	/* 103-199 unassigned */
	{ 200,	"OK" },
	{ 201,	"Created" },
	{ 202,	"Accepted" },
	{ 203,	"Non-Authoritative Information" },
	{ 204,	"No Content" },
	{ 205,	"Reset Content" },
	{ 206,	"Partial Content" },
	{ 207,	"Multi-Status" },
	{ 208,	"Already Reported" },
	/* 209-225 unassigned */
	{ 226,	"IM Used" },
	/* 227-299 unassigned */
	{ 300,	"Multiple Choices" },
	{ 301,	"Moved Permanently" },
	{ 302,	"Found" },
	{ 303,	"See Other" },
	{ 304,	"Not Modified" },
	{ 305,	"Use Proxy" },
	{ 306,	"Switch Proxy" },
	{ 307,	"Temporary Redirect" },
	{ 308,	"Permanent Redirect" },
	/* 309-399 unassigned */
	{ 400,	"Bad Request" },
	{ 401,	"Unauthorized" },
	{ 402,	"Payment Required" },
	{ 403,	"Forbidden" },
	{ 404,	"Not Found" },
	{ 405,	"Method Not Allowed" },
	{ 406,	"Not Acceptable" },
	{ 407,	"Proxy Authentication Required" },
	{ 408,	"Request Timeout" },
	{ 409,	"Conflict" },
	{ 410,	"Gone" },
	{ 411,	"Length Required" },
	{ 412,	"Precondition Failed" },
	{ 413,	"Payload Too Large" },
	{ 414,	"URI Too Long" },
	{ 415,	"Unsupported Media Type" },
	{ 416,	"Range Not Satisfiable" },
	{ 417,	"Expectation Failed" },
	{ 418,	"I'm a teapot" },
	/* 419-421 unassigned */
	{ 420,	"Enhance Your Calm" },
	{ 422,	"Unprocessable Entity" },
	{ 423,	"Locked" },
	{ 424,	"Failed Dependency" },
	/* 425 unassigned */
	{ 426,	"Upgrade Required" },
	/* 427 unassigned */
	{ 428,	"Precondition Required" },
	{ 429,	"Too Many Requests" },
	/* 430 unassigned */
	{ 431,	"Request Header Fields Too Large" },
	/* 432-450 unassigned */
	{ 451,	"Unavailable For Legal Reasons" },
	/* 452-499 unassigned */
	{ 500,	"Internal Server Error" },
	{ 501,	"Not Implemented" },
	{ 502,	"Bad Gateway" },
	{ 503,	"Service Unavailable" },
	{ 504,	"Gateway Timeout" },
	{ 505,	"HTTP Version Not Supported" },
	{ 506,	"Variant Also Negotiates" },
	{ 507,	"Insufficient Storage" },
	{ 508,	"Loop Detected" },
	/* 509 unassigned */
	{ 510,	"Not Extended" },
	{ 511,	"Network Authentication Required" },
	/* 512-599 unassigned */
	{ 0,	NULL },
};

struct http_headers {
	char	*location;
	off_t	 content_length;
	int	 chunked;
	int	 retry_after;
};

static void		 decode_chunk(int, uint, FILE *);
static char		*header_lookup(const char *, const char *);
static const char	*http_error(int);
static int		 http_status_cmp(const void *, const void *);
static void		 http_headers_free(struct http_headers *);
static ssize_t		 http_getline(int, char **, size_t *);
static void		 http_proxy_connect(struct url *, struct url *);
static char		*http_prepare_request(struct url *, off_t *);
static size_t		 http_read(int, char *, size_t);
static struct url	*http_redirect(struct url *, char *);
static void		 http_copy_chunks(struct url *, FILE *, off_t *);
static int		 http_request(int, const char *,
			    struct http_headers **);
static char		*relative_path_resolve(const char *, const char *);

#ifndef NOSSL
static void		 tls_copy_file(struct url *, FILE *, off_t *);
static ssize_t		 tls_getline(char **, size_t *, struct tls *);
#endif /* NOSSL */

static FILE		*fp;
static int		 chunked;

void
http_connect(struct url *url, int timeout)
{
	static struct url	*proxy;
	const char		*host, *port;
	int			 sock;

	proxy = (url->scheme == S_HTTPS || url->scheme == S_HTTP) ?
	    http_proxy : ftp_proxy;

	host = proxy ? proxy->host : url->host;
	port = proxy ? proxy->port : url->port;
	if ((sock = tcp_connect(host, port, timeout)) == -1)
		exit(1);

	if ((fp = fdopen(sock, "r+")) == NULL)
		err(1, "%s: fdopen", __func__);

	if (proxy)
		http_proxy_connect(proxy, url);

#ifndef NOSSL
	if (url->scheme != S_HTTPS)
		return;

	if ((ctx = tls_client()) == NULL)
		errx(1, "failed to create tls client");

	if (tls_configure(ctx, tls_config) != 0)
		errx(1, "%s: %s", __func__, tls_error(ctx));

	if (tls_connect_socket(ctx, fileno(fp), url->host) != 0)
		errx(1, "%s: %s", __func__, tls_error(ctx));
#endif /* NOSSL */
}

static void
http_proxy_connect(struct url *proxy, struct url *url)
{
	struct http_headers	*headers;
	char			*auth = NULL, *req;
	int			 authlen = 0, code;

	if (proxy->basic_auth) {
		authlen = xasprintf(&auth,
		    "Proxy-Authorization: Basic %s\r\n", proxy->basic_auth);
	}

	xasprintf(&req,
	    "CONNECT %s:%s HTTP/1.0\r\n"
	    "User-Agent: %s\r\n"
	    "%s"
	    "\r\n",
	    url->host, url->port,
	    useragent,
	    proxy->basic_auth ? auth : "");

	freezero(auth, authlen);
	if ((code = http_request(S_HTTP, req, &headers)) != 200)
		errx(1, "%s: Failed to CONNECT to %s:%s: %d %s",
			__func__, url->host, url->port, code, http_error(code));

	free(req);
	http_headers_free(headers);
}

static char *
http_prepare_request(struct url *url, off_t *offset)
{
	char	*auth = NULL, *path = NULL, *range = NULL, *req;
	int	 authlen = 0;

	if (*offset)
		xasprintf(&range, "Range: bytes=%lld-\r\n", *offset);

	if (url->basic_auth) {
		authlen = xasprintf(&auth,
		    "Authorization: Basic %s\r\n", url->basic_auth);
	}

	if (url->path)
		path = url_encode(url->path);

	xasprintf(&req,
	    "GET %s HTTP/1.1\r\n"
	    "Host: %s\r\n"
	    "%s"
	    "%s"
	    "Connection: close\r\n"
	    "User-Agent: %s\r\n"
	    "\r\n",
	    path ? path : "/",
	    url->host,
	    *offset ? range : "",
	    url->basic_auth ? auth : "",
	    useragent);

	free(range);
	freezero(auth, authlen);
	free(path);
	return req;
}

struct url *
http_get(struct url *url, off_t *offset, off_t *sz)
{
	struct http_headers	*headers;
	char			*req;
	int			 code, redirects = 0, retry = 0;

	do {
		log_request("Requesting", url, http_proxy);
		req = http_prepare_request(url, offset);
		code = http_request(url->scheme, req, &headers);
		free(req);
		switch (code) {
		case 200:
			if (*offset) {
				warnx("Server does not support resume.");
				*offset = 0;
			}
			break;
		case 206:
			break;
		case 301:
		case 302:
		case 303:
		case 307:
			http_close(url);
			if (++redirects > MAX_REDIRECTS)
				errx(1, "Too many redirections requested.");

			if (headers->location == NULL) {
				errx(1,
				    "%s: Location header missing", __func__);
			}

			url = http_redirect(url, headers->location);
			http_headers_free(headers);
			log_request("Redirected to", url, http_proxy);
			http_connect(url, 0);
			break;
		case 416:
			errx(1, "File is already fully retrieved.");
			break;
		case 503:
			if (headers->retry_after == 0 && retry == 0) {
				http_close(url);
				http_headers_free(headers);
				retry = 1;
				log_request("Retrying", url, http_proxy);
				http_connect(url, 0);
				break;
			}
			/* FALLTHROUGH */
		default:
			errx(1, "Error retrieving file: %d %s",
			    code, http_error(code));
		}
	} while (code == 301 || code == 302 ||
	    code == 303 || code == 307 || code == 503);

	*sz = headers->content_length + *offset;
	chunked = headers->chunked;
	http_headers_free(headers);
	return url;
}

void
http_save(struct url *url, FILE *dst_fp, off_t *offset)
{
	if (chunked)
		http_copy_chunks(url, dst_fp, offset);
#ifndef NOSSL
	else if (url->scheme == S_HTTPS)
		tls_copy_file(url, dst_fp, offset);
#endif /* NOSSL */
	else
		copy_file(dst_fp, fp, offset);
}

static struct url *
http_redirect(struct url *old_url, char *location)
{
	struct url	*new_url;

	/* absolute uri reference */
	if (strncasecmp(location, "http", 4) == 0 ||
	    strncasecmp(location, "https", 5) == 0) {
		new_url = xurl_parse(location);
		goto done;
	}

	/* relative uri reference */
	new_url = xcalloc(1, sizeof *new_url);
	new_url->scheme = old_url->scheme;
	new_url->host = xstrdup(old_url->host);
	new_url->port = xstrdup(old_url->port);

	/* absolute-path reference */
	if (location[0] == '/')
		new_url->path = xstrdup(location);
	else
		new_url->path = relative_path_resolve(old_url->path, location);

 done:
	url_free(old_url);
	return new_url;
}

static char *
relative_path_resolve(const char *base_path, const char *location)
{
	char	*new_path, *p;

	/* trim fragment component from both uri */
	if ((p = strchr(location, '#')) != NULL)
		*p = '\0';
	if (base_path && (p = strchr(base_path, '#')) != NULL)
		*p = '\0';

	if (base_path == NULL)
		xasprintf(&new_path, "/%s", location);
	else if (base_path[strlen(base_path) - 1] == '/')
		xasprintf(&new_path, "%s%s", base_path, location);
	else {
		p = dirname(base_path);
		xasprintf(&new_path, "%s/%s",
		    strcmp(p, ".") == 0 ? "" : p, location);
	}

	return new_path;
}

static void
http_copy_chunks(struct url *url, FILE *dst_fp, off_t *offset)
{
	char	*buf = NULL;
	size_t	 n = 0;
	uint	 chunk_sz;

	http_getline(url->scheme, &buf, &n);
	if (sscanf(buf, "%x", &chunk_sz) != 1)
		errx(1, "%s: Failed to get chunk size", __func__);

	while (chunk_sz > 0) {
		decode_chunk(url->scheme, chunk_sz, dst_fp);
		*offset += chunk_sz;
		http_getline(url->scheme, &buf, &n);
		if (sscanf(buf, "%x", &chunk_sz) != 1)
			errx(1, "%s: Failed to get chunk size", __func__);
	}

	free(buf);
}

static void
decode_chunk(int scheme, uint sz, FILE *dst_fp)
{
	size_t	bufsz;
	size_t	r;
	char	buf[BUFSIZ], crlf[2];

	bufsz = sizeof(buf);
	while (sz > 0) {
		if (sz < bufsz)
			bufsz = sz;

		r = http_read(scheme, buf, bufsz);
		if (fwrite(buf, 1, r, dst_fp) != r)
			errx(1, "%s: fwrite", __func__);

		sz -= r;
	}

	/* CRLF terminating the chunk */
	if (http_read(scheme, crlf, sizeof(crlf)) != sizeof(crlf))
		errx(1, "%s: Failed to read terminal crlf", __func__);

	if (crlf[0] != '\r' || crlf[1] != '\n')
		errx(1, "%s: Invalid chunked encoding", __func__);
}

void
http_close(struct url *url)
{
#ifndef NOSSL
	ssize_t	r;

	if (url->scheme == S_HTTPS) {
		if (tls_session_fd != -1)
			dprintf(STDERR_FILENO, "tls session resumed: %s\n",
			    tls_conn_session_resumed(ctx) ? "yes" : "no");

		do {
			r = tls_close(ctx);
		} while (r == TLS_WANT_POLLIN || r == TLS_WANT_POLLOUT);
		tls_free(ctx);
	}

#endif /* NOSSL */
	fclose(fp);
	chunked = 0;
}

static int
http_request(int scheme, const char *req, struct http_headers **hdrs)
{
	struct http_headers	*headers;
	const char		*e;
	char			*buf = NULL, *p;
	size_t			 n = 0;
	ssize_t			 buflen;
	uint			 code;
#ifndef NOSSL
	size_t			 len;
	ssize_t			 nw;
#endif /* NOSSL */

	if (io_debug)
		fprintf(stderr, "<<< %s", req);

	switch (scheme) {
#ifndef NOSSL
	case S_HTTPS:
		len = strlen(req);
		while (len > 0) {
			nw = tls_write(ctx, req, len);
			if (nw == TLS_WANT_POLLIN || nw == TLS_WANT_POLLOUT)
				continue;
			if (nw < 0)
				errx(1, "tls_write: %s", tls_error(ctx));
			req += nw;
			len -= nw;
		}
		break;
#endif /* NOSSL */
	case S_HTTP:
		if (fprintf(fp, "%s", req) < 0)
			errx(1, "%s: fprintf", __func__);
		(void)fflush(fp);
		break;
	}

	http_getline(scheme, &buf, &n);
	if (io_debug)
		fprintf(stderr, ">>> %s", buf);

	if (sscanf(buf, "%*s %u %*s", &code) != 1)
		errx(1, "%s: failed to extract status code", __func__);

	if (code < 100 || code > 511)
		errx(1, "%s: invalid status code %d", __func__, code);

	headers = xcalloc(1, sizeof *headers);
	headers->retry_after = -1;
	for (;;) {
		buflen = http_getline(scheme, &buf, &n);
		buflen -= 1;
		if (buflen > 0 && buf[buflen - 1] == '\r')
			buflen -= 1;
		buf[buflen] = '\0';

		if (io_debug)
			fprintf(stderr, ">>> %s\n", buf);

		if (buflen == 0)
			break; /* end of headers */

		if ((p = header_lookup(buf, "Content-Length:")) != NULL) {
			headers->content_length = strtonum(p, 0, INT64_MAX, &e);
			if (e)
				err(1, "%s: Content-Length is %s: %lld",
				    __func__, e, headers->content_length);
		}

		if ((p = header_lookup(buf, "Location:")) != NULL)
			headers->location = xstrdup(p);

		if ((p = header_lookup(buf, "Transfer-Encoding:")) != NULL)
			if (strcasestr(p, "chunked") != NULL)
				headers->chunked = 1;

		if ((p = header_lookup(buf, "Retry-After:")) != NULL) {
			headers->retry_after = strtonum(p, 0, 0, &e);
			if (e)
				headers->retry_after = -1;
		}
	}

	*hdrs = headers;
	free(buf);
	return code;
}

static void
http_headers_free(struct http_headers *headers)
{
	if (headers == NULL)
		return;

	free(headers->location);
	free(headers);
}

static char *
header_lookup(const char *buf, const char *key)
{
	char	*p;

	if (strncasecmp(buf, key, strlen(key)) == 0) {
		if ((p = strchr(buf, ' ')) == NULL)
			errx(1, "Failed to parse %s", key);
		return ++p;
	}

	return NULL;
}

static ssize_t
http_getline(int scheme, char **buf, size_t *n)
{
	ssize_t	buflen;

	switch (scheme) {
#ifndef NOSSL
	case S_HTTPS:
		if ((buflen = tls_getline(buf, n, ctx)) == -1)
			errx(1, "%s: tls_getline", __func__);
		break;
#endif /* NOSSL */
	case S_HTTP:
		if ((buflen = getline(buf, n, fp)) == -1)
			err(1, "%s: getline", __func__);
		break;
	default:
		errx(1, "%s: invalid scheme", __func__);
	}

	return buflen;
}

static size_t
http_read(int scheme, char *buf, size_t size)
{
	size_t	r;
#ifndef NOSSL
	ssize_t	rs;
#endif /* NOSSL */

	switch (scheme) {
#ifndef NOSSL
	case S_HTTPS:
		do {
			rs = tls_read(ctx, buf, size);
		} while (rs == TLS_WANT_POLLIN || rs == TLS_WANT_POLLOUT);
		if (rs == -1)
			errx(1, "%s: tls_read: %s", __func__, tls_error(ctx));
		r = rs;
		break;
#endif /* NOSSL */
	case S_HTTP:
		if ((r = fread(buf, 1, size, fp)) < size)
			if (!feof(fp))
				errx(1, "%s: fread", __func__);
		break;
	default:
		errx(1, "%s: invalid scheme", __func__);
	}

	return r;
}

static const char *
http_error(int code)
{
	struct http_status	error, *res;

	/* Set up key */
	error.code = code;

	if ((res = bsearch(&error, http_status,
	    sizeof(http_status) / sizeof(http_status[0]) - 1,
	    sizeof(http_status[0]), http_status_cmp)) != NULL)
		return (res->name);

	return (NULL);
}

static int
http_status_cmp(const void *a, const void *b)
{
	const struct http_status *ea = a;
	const struct http_status *eb = b;

	return (ea->code - eb->code);
}

#ifndef NOSSL
void
https_init(char *tls_options)
{
	char		*str;
	int		 depth;
	const char	*ca_file, *errstr;

	if (tls_init() != 0)
		errx(1, "tls_init failed");

	if ((tls_config = tls_config_new()) == NULL)
		errx(1, "tls_config_new failed");

	if (tls_config_set_protocols(tls_config, TLS_PROTOCOLS_ALL) != 0)
		errx(1, "tls set protocols failed: %s",
		    tls_config_error(tls_config));

	if (tls_config_set_ciphers(tls_config, "legacy") != 0)
		errx(1, "tls set ciphers failed: %s",
		    tls_config_error(tls_config));

	ca_file = tls_default_ca_cert_file();
	while (tls_options && *tls_options) {
		switch (getsubopt(&tls_options, tls_verify_opts, &str)) {
		case HTTP_TLS_CAFILE:
			if (str == NULL)
				errx(1, "missing CA file");
			ca_file = str;
			break;
		case HTTP_TLS_CAPATH:
			if (str == NULL)
				errx(1, "missing ca path");
			if (tls_config_set_ca_path(tls_config, str) != 0)
				errx(1, "tls ca path failed");
			break;
		case HTTP_TLS_CIPHERS:
			if (str == NULL)
				errx(1, "missing cipher list");
			if (tls_config_set_ciphers(tls_config, str) != 0)
				errx(1, "tls set ciphers failed");
			break;
		case HTTP_TLS_DONTVERIFY:
			tls_config_insecure_noverifycert(tls_config);
			tls_config_insecure_noverifyname(tls_config);
			break;
		case HTTP_TLS_VERIFYDEPTH:
			if (str == NULL)
				errx(1, "missing depth");
			depth = strtonum(str, 0, INT_MAX, &errstr);
			if (errstr)
				errx(1, "Cert validation depth is %s", errstr);
			tls_config_set_verify_depth(tls_config, depth);
			break;
		case HTTP_TLS_MUSTSTAPLE:
			tls_config_ocsp_require_stapling(tls_config);
			break;
		case HTTP_TLS_NOVERIFYTIME:
			tls_config_insecure_noverifytime(tls_config);
			break;
		case HTTP_TLS_SESSION:
			if (str == NULL)
				errx(1, "missing session file");
			tls_session_fd = open(str, O_RDWR|O_CREAT, 0600);
			if (tls_session_fd == -1)
				err(1, "failed to open or create session file "
				    "'%s'", str);
			if (tls_config_set_session_fd(tls_config,
			    tls_session_fd) == -1)
				errx(1, "failed to set session: %s",
				    tls_config_error(tls_config));
			break;
		case HTTP_TLS_DOVERIFY:
			/* For compatibility, we do verify by default */
			break;
		default:
			errx(1, "Unknown -S suboption `%s'",
			    suboptarg ? suboptarg : "");
		}
	}

	if (tls_config_set_ca_file(tls_config, ca_file) == -1)
		errx(1, "tls_config_set_ca_file failed");
}

static ssize_t
tls_getline(char **buf, size_t *buflen, struct tls *tls)
{
	char		*newb;
	size_t		 newlen, off;
	int		 ret;
	unsigned char	 c;

	if (buf == NULL || buflen == NULL)
		return -1;

	/* If buf is NULL, we have to assume a size of zero */
	if (*buf == NULL)
		*buflen = 0;

	off = 0;
	do {
		do {
			ret = tls_read(tls, &c, 1);
		} while (ret == TLS_WANT_POLLIN || ret == TLS_WANT_POLLOUT);
		if (ret == -1)
			return -1;

		/* Ensure we can handle it */
		if (off + 2 > SSIZE_MAX)
			return -1;

		newlen = off + 2; /* reserve space for NUL terminator */
		if (newlen > *buflen) {
			newlen = newlen < MINBUF ? MINBUF : *buflen * 2;
			newb = recallocarray(*buf, *buflen, newlen, 1);
			if (newb == NULL)
				return -1;

			*buf = newb;
			*buflen = newlen;
		}

		*(*buf + off) = c;
		off += 1;
	} while (c != '\n');

	*(*buf + off) = '\0';
	return off;
}

static void
tls_copy_file(struct url *url, FILE *dst_fp, off_t *offset)
{
	char	*tmp_buf;
	ssize_t	 r;

	tmp_buf = xmalloc(TMPBUF_LEN);
	for (;;) {
		do {
			r = tls_read(ctx, tmp_buf, TMPBUF_LEN);
		} while (r == TLS_WANT_POLLIN || r == TLS_WANT_POLLOUT);

		if (r == -1)
			errx(1, "%s: tls_read: %s", __func__, tls_error(ctx));
		else if (r == 0)
			break;

		*offset += r;
		if (fwrite(tmp_buf, 1, r, dst_fp) != (size_t)r)
			err(1, "%s: fwrite", __func__);
	}
	free(tmp_buf);
}
#endif /* NOSSL */