From 28c32424d04f982985b685ce891ed7db6237504e Mon Sep 17 00:00:00 2001 From: Nathan Wagner Date: Wed, 13 Feb 2019 09:09:43 +0000 Subject: [PATCH] support file urls --- crypto/buffer.c | 4 ++ crypto/buffer.h | 1 + crypto/tlse.c | 36 ++++++---- doc/zpm-fetchurl.8 | 73 +++++++++++++++++++ doc/zpm.8 | 1 + zpm-fetchurl.c | 176 ++++++++++++++++++++++++++++++++------------- 6 files changed, 230 insertions(+), 61 deletions(-) create mode 100644 doc/zpm-fetchurl.8 diff --git a/crypto/buffer.c b/crypto/buffer.c index a65c21e..790fb2c 100644 --- a/crypto/buffer.c +++ b/crypto/buffer.c @@ -94,6 +94,10 @@ void tls_buffer_append(struct tls_buffer *buffer, const unsigned char *bytes, si buffer->len += n; } +void tls_buffer_append_str(struct tls_buffer *buf, const unsigned char *s) { + tls_buffer_append(buf, s, strlen(s)); +} + void tls_buffer_append16(struct tls_buffer *buffer, uint16_t n) { tls_buffer_expand(buffer, 2); diff --git a/crypto/buffer.h b/crypto/buffer.h index ad37f67..c1ace92 100644 --- a/crypto/buffer.h +++ b/crypto/buffer.h @@ -36,6 +36,7 @@ void tls_buffer_free(struct tls_buffer *b); void tls_buffer_set(struct tls_buffer *buffer, int ch); void tls_buffer_compact(struct tls_buffer *b); void tls_buffer_append(struct tls_buffer *b, const unsigned char *bytes, size_t n); +void tls_buffer_append_str(struct tls_buffer *b, const unsigned char *bytes); void tls_buffer_append16(struct tls_buffer *b, uint16_t n); void tls_buffer_append24(struct tls_buffer *b, uint32_t n); void tls_buffer_append_byte(struct tls_buffer *b, uint8_t n); diff --git a/crypto/tlse.c b/crypto/tlse.c index 8920467..233fe26 100644 --- a/crypto/tlse.c +++ b/crypto/tlse.c @@ -61,6 +61,8 @@ #define CHECK_HANDSHAKE_STATE(context, n, limit) { if (context->hs_messages[n] >= limit) { DEBUG_PRINT("* UNEXPECTED MESSAGE (%i)\n", (int)n); payload_res = TLS_UNEXPECTED_MESSAGE; break; } context->hs_messages[n]++; } +//#define MARK fprintf(stderr, "%s %s:%d\n", __FILE__, __func__, __LINE__) +#define MARK typedef enum { KEA_dhe_dss, KEA_dhe_rsa, @@ -2441,16 +2443,7 @@ int tls_random(unsigned char *key, int len) { } int tls_established(struct TLSContext *context) { - if (context) { - if (context->critical_error) { - return -1; - } - - if (context->connection_status == TLS_CONNECTED) { - return 1; - } - } - return 0; + return context && context->connection_status == TLS_CONNECTED; } void tls_read_clear(struct TLSContext *context) { @@ -5567,21 +5560,35 @@ int tls_connect(struct TLSContext *context) { int res; ssize_t read_size; - if (!context || context->fd <= 0 || context->critical_error) { + MARK; + if (!context || context->fd < 0 || context->critical_error) { + if (!context) { + MARK; + } else if (context->fd < 0) { + MARK; + } else { + MARK; + } + return TLS_GENERIC_ERROR; } + MARK; if (context->is_server) { return TLS_UNEXPECTED_MESSAGE; } + MARK; res = tls_queue_packet(tls_build_client_hello(context)); + MARK; if (res < 0) { return res; } + MARK; res = tls_fsync(context); + MARK; if (res < 0) { return res; } @@ -5593,13 +5600,18 @@ int tls_connect(struct TLSContext *context) { return res; } } + MARK; if (tls_established(context)) { return 1; } + MARK; if (context->critical_error) { + fprintf(stderr, "critical error: %d\n", + context->critical_error); return TLS_GENERIC_ERROR; } } + MARK; return read_size; } @@ -5677,7 +5689,7 @@ ssize_t tls_read(struct TLSContext *context, void *buf, size_t count) { return TLS_GENERIC_ERROR; } - if (tls_established(context) != 1) { + if (!tls_established(context)) { return TLS_GENERIC_ERROR; } diff --git a/doc/zpm-fetchurl.8 b/doc/zpm-fetchurl.8 new file mode 100644 index 0000000..c6f748c --- /dev/null +++ b/doc/zpm-fetchurl.8 @@ -0,0 +1,73 @@ +.TH zpm-fetchurl 8 2018-12-09 "ZPM 0.4" +.SH NAME +zpm-fetchurl \- download files +.SH SYNOPSIS +.B zpm fetchurl +[ +.B -ISkKr +] +[ +.BI -o file +] +.I [ url ] +.SH DESCRIPTION +\fBzpm-fetchurl\fR downloads files +.PP +\fBzpm-fetchurl\fR understands http, https, and file type urls. +For https, it uses a trust on first use model for ssl certificates, but a more +common model verifying via root certificates, or no verfication can also be +used. +.PP +While this program can be used directly, it is intended for use by zpm scripts +for downloading repositories and packages without requiring an external +dependency. +.SH OPTIONS +.TP +.B \-I +output the response header only, implies -r +.TP +.B \-r +output the entire response header, including the header +.TP +.B \-S +output the response status code only +.TP +.B \-k +Don't verify any certificates. +.TP +.B \-K +Verify certificates via root certificates. This is normally the default for +TLS, but zpm-fetchurl uses a trust on first use policy instead. Root +certificates are found by default in /etc/zpm/roots.pem, but the ZPM_ROOTFILE +environment variable can be used to override this. +.TP +.BI \-R " limit" +set the limit on following redirects, defaults to 50 +.TP +.BI \-z " path" +Only download if remote is newer than the file given by path +.TP +.BI \-o " path" +Write the output to the file given by \fIpath\fR. Output is +written to stdout by default. +.TP +.B \-f +Fail silently on errors. +.TP +.B \-# +Output a progress bar. +.SH EXAMPLES +.TP +zpm fetchurl -o zpm-0.1.2.zpm https://zoranix.net/repo/packages/zpm-0.1.2.zpm +.SH EXIT STATUS +0 on success non zero on failure +.SH FILES +/var/lib/zpm/known_hosts +~/.zpm/known_hosts +.SH ENVIRONMENT +ZPM_KNOWNHOSTS +ZPM_ROOTFILE +.SH AUTHOR +Nathan Wagner +.SH SEE ALSO +zpm(8) diff --git a/doc/zpm.8 b/doc/zpm.8 index 6fa512e..cb36c21 100644 --- a/doc/zpm.8 +++ b/doc/zpm.8 @@ -99,3 +99,4 @@ the musl copyright. .BR zpm-repo (8) .BR zpm-quote (8) .BR zpm-hash (8) +.BR zpm-fetchurl (8) diff --git a/zpm-fetchurl.c b/zpm-fetchurl.c index 4ffca23..1b33724 100644 --- a/zpm-fetchurl.c +++ b/zpm-fetchurl.c @@ -15,6 +15,7 @@ #include #include "tlse.h" +#define MARK fprintf(stderr, "%s %s:%d\n", __FILE__, __func__, __LINE__) struct tls_uri { char *scheme; @@ -132,29 +133,38 @@ int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int if (geteuid() == 0) { trustfile = "/var/lib/zpm/known_hosts"; } else { - homedir = getenv("HOME"); - if (!homedir) { + /* we could do this with a series of + * openat() calls instead of building + * up a string + */ + trustfile = getenv("HOME"); + if (!trustfile) { + fprintf(stderr, "home = %s\n", trustfile); return 1; } - len = snprintf(homedir, 0, "%s/.zpm/known_hosts", homedir); + len = snprintf(homedir, 0, "%s/.zpm/known_hosts", trustfile); homedir = malloc(len+1); if (!homedir) { return 1; } - len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", homedir); + len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", trustfile); trustfile = homedir; } } /* cert is valid on its face, so check against the trust db */ trustdb = open(trustfile, O_RDWR|O_CREAT, 0600); - if (homedir) { - free(homedir); - } if (trustdb == -1) { - fprintf(stderr, "cannot open trustdb: %s\n", strerror(errno)); + fprintf(stderr, "cannot open trustdb %s: %s\n", trustfile, strerror(errno)); + if (homedir) { + free(homedir); + } return 1; } + if (homedir) { + free(homedir); + } + len = 0; tls_buffer_init(&tbuf, 128); do { @@ -205,10 +215,17 @@ int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int /* got here, so we should be at EOF, so add this host to trust db */ lseek(trustdb, 0, SEEK_END); - write(trustdb, certhash, 64); - write(trustdb, ":", 1); - write(trustdb, context->sni, strlen(context->sni)); - write(trustdb, "\n", 1); + + /* re-use the buffer so we only do one write */ + /* ignore errors, the cert is fine, we just can't update + * the trustdb if there's errors here + */ + tbuf.len = 0; + tls_buffer_append(&tbuf, certhash, 64); + tls_buffer_append_byte(&tbuf, ':'); + tls_buffer_append(&tbuf, context->sni, strlen(context->sni)); + tls_buffer_append_byte(&tbuf, '\n'); + write(trustdb, tbuf.buffer, tbuf.len); close(trustdb); tls_buffer_free(&tbuf); @@ -222,17 +239,13 @@ int verify_roots(struct TLSContext *context, struct TLSCertificate **chain, int if (chain) { for (i = 0; i < len; i++) { struct TLSCertificate *certificate = chain[i]; - // check validity date err = tls_certificate_is_valid(certificate); if (err) { return err; } - // check certificate in certificate->bytes of length - // certificate->len the certificate is in ASN.1 DER - // format } } - // check if chain is valid + err = tls_certificate_chain_is_valid(chain, len); if (err) { return err; @@ -280,11 +293,9 @@ int month(char *m) { /* Wed, 06 Feb 2019 10:06:05 GMT */ time_t parse_date(char *d) { - //static const char format[] = "%a, %d %b %Y %H:%M:%S %Z"; // rfc 1123 struct tm tm = { 0 }; int rv; - //char *data = "Tue, 13 Dec 2011 16:08:21 GMT"; int h, m, s, dom, Y; char M[4]; rv = sscanf(d, "%*[a-zA-Z,] %d %s %d %d:%d:%d", &dom, M, &Y, &h, &m, &s); @@ -425,6 +436,16 @@ void append_header(struct tls_buffer *buf, char *header, char *val) { tls_buffer_append(buf, "\r\n", 2); } +void append_timeheader(struct tls_buffer *buf, char *header, time_t ts) { + char timestr[80]; + struct tm *tm; + + tm = gmtime(&ts); + + strftime(timestr, sizeof timestr, "%a, %d %b %Y %H:%M:%S GMT", tm); + append_header(buf, header, timestr); +} + static void pdots(int len, int ch, unsigned long was, unsigned long now, unsigned long total) { was = len * was / total; @@ -437,6 +458,56 @@ static void pdots(int len, int ch, unsigned long was, unsigned long now, } } +static void fake_header(struct io *io, int fd) { + struct stat st; + int code = 200, rv; + char *message, codestr[5], length[32]; + struct tls_buffer *hdr = &io->response; + + if (fd == -1) { + switch (errno) { + case EACCES: code = 403; break; + case ENOENT: code = 404; break; + default: code = 500; break; + } + } else { + rv = fstat(fd, &st); + if (rv == -1) { + code = 500; + } + } + + if (io->last_modified >= st.st_mtime) { + code = 304; + } + + switch (code) { + case 200: message = "OK"; break; + case 304: message = "Not Modified"; break; + case 403: message = "Forbidden"; break; + case 404: message = "Not Found"; break; + case 500: message = "Internal Server Error"; break; + default: break; + } + sprintf(codestr, "%0.3d ", code); + tls_buffer_append(hdr, "HTTP/1.1 ", 9); + tls_buffer_append(hdr, codestr, 4); + tls_buffer_append_str(hdr, message); + tls_buffer_append(hdr, "\r\n", 2); + + append_timeheader(hdr, "Date", time(NULL)); + append_header(hdr, "Server", "zpm-fetchurl/0.9"); + if (code < 400) { + append_timeheader(hdr, "Last-Modified", st.st_mtime); + sprintf(length, "%zu", st.st_size); + append_header(hdr, "Content-Length", length); + } + + append_header(hdr, "Connection", "close"); + append_header(hdr, "Content-Type", "application/octet-stream"); + tls_buffer_append(hdr, "\r\n", 2); +} + int main(int ac, char *av[]) { int sockfd, port = -1, rv; ssize_t ret; @@ -492,6 +563,7 @@ int main(int ac, char *av[]) { exit(EXIT_FAILURE); } + io.last_modified = 0; if (lmfile) { struct stat st; int rv; @@ -504,6 +576,7 @@ int main(int ac, char *av[]) { exit(EXIT_FAILURE); } ts = st.st_mtime; + io.last_modified = ts; mtime = gmtime(&ts); strftime(lmtime, sizeof lmtime, "%a, %d %b %Y %H:%M:%S GMT", mtime); } @@ -543,8 +616,7 @@ int main(int ac, char *av[]) { tls_buffer_append(&request, "GET ", 4); } tls_buffer_append(&request, uri.path, strlen(uri.path)); - tls_buffer_append(&request, " HTTP/1.1", 9); - tls_buffer_append(&request, "\r\n", 2); + tls_buffer_append(&request, " HTTP/1.1\r\n", 11); append_header(&request, "Host", host); append_header(&request, "Connection", "close"); @@ -552,7 +624,6 @@ int main(int ac, char *av[]) { append_header(&request, "If-Modified-Since", lmtime); } tls_buffer_append(&request, "\r\n", 2); - //fprintf(stderr, "msg =\n%.*s", (int)request.len, request.buffer); if (!strcmp(uri.scheme, "https")) { use_tls = 1; @@ -589,41 +660,49 @@ int main(int ac, char *av[]) { tls_sni_set(clientssl, uri.host); clientssl->sync = 1; io.tls = clientssl; - } - - sockfd = open_tcp_connection(host, port); - io.socket = sockfd; - - if (sockfd < 0) { - exit(EXIT_FAILURE); - } - - if (use_tls) { + sockfd = open_tcp_connection(host, port); + if (sockfd < 0) { + perror("can't open connection"); + exit(EXIT_FAILURE); + } tls_set_fd(clientssl, sockfd); - if ((rv = tls_connect(clientssl)) != 1) { fprintf(stderr, "Handshake Error %i\n", rv); - return -5; + return 1; } - ret = tls_write(clientssl, request.buffer, request.len); + } else if (!strcmp(uri.scheme, "http")) { + sockfd = open_tcp_connection(host, port); + if (sockfd < 0) { + perror("can't open connection"); + exit(EXIT_FAILURE); + } + ret = write(sockfd, request.buffer, request.len); + } else if (!strcmp(uri.scheme, "file")) { + sockfd = open(uri.path, O_RDONLY); + fake_header(&io, sockfd); + ret = 0; } else { - ret = write(io.socket, request.buffer, request.len); + fprintf(stderr, "scheme %s unknown\n", uri.scheme); + exit(EXIT_FAILURE); } - if (ret < 0) { - fprintf(stderr, "write error %zd\n", ret); - return -6; + if (ret == -1) { + fprintf(stderr, "unable to write http request: %s\n", strerror(errno)); + exit(EXIT_FAILURE); } + io.socket = sockfd; + do { - ret = fill_buffer(&io); - if (ret < 0) { - break; + if (io.response.len >= 4) { + eoh = strstr(io.response.buffer, "\r\n\r\n"); } - eoh = strstr(io.response.buffer, "\r\n\r\n"); - if (ret == 0) { - break; + if (!eoh) { + ret = fill_buffer(&io); + if (ret <= 0) { + break; + } } } while (!eoh); @@ -652,10 +731,6 @@ int main(int ac, char *av[]) { break; } - if (io.status_code != 200) { - break; - } - if (!raw) { tls_buffer_shift(&io.response, header_len); } @@ -687,6 +762,9 @@ int main(int ac, char *av[]) { } total += ret; } + if (head) { + break; + } ret = fill_buffer(&io); } while (ret > 0); -- 2.40.0