]> pd.if.org Git - zpackage/commitdiff
support file urls
authorNathan Wagner <nw@hydaspes.if.org>
Wed, 13 Feb 2019 09:09:43 +0000 (09:09 +0000)
committerNathan Wagner <nw@hydaspes.if.org>
Wed, 13 Feb 2019 09:09:43 +0000 (09:09 +0000)
crypto/buffer.c
crypto/buffer.h
crypto/tlse.c
doc/zpm-fetchurl.8 [new file with mode: 0644]
doc/zpm.8
zpm-fetchurl.c

index a65c21ed2e3dd0a8fa66d67287460be5fd6a6d6c..790fb2ce7e0a0cc2f5a47df4f1cfe692b9adb424 100644 (file)
@@ -94,6 +94,10 @@ void tls_buffer_append(struct tls_buffer *buffer, const unsigned char *bytes, si
        buffer->len += n;
 }
 
+void tls_buffer_append_str(struct tls_buffer *buf, const unsigned char *s) {
+       tls_buffer_append(buf, s, strlen(s));
+}
+
 void tls_buffer_append16(struct tls_buffer *buffer, uint16_t n) {
        tls_buffer_expand(buffer, 2);
 
index ad37f67ff22e2a1811e58bb23ef061a1562da666..c1ace92db1bf65914894966e063df9a955fcb126 100644 (file)
@@ -36,6 +36,7 @@ void tls_buffer_free(struct tls_buffer *b);
 void tls_buffer_set(struct tls_buffer *buffer, int ch);
 void tls_buffer_compact(struct tls_buffer *b);
 void tls_buffer_append(struct tls_buffer *b, const unsigned char *bytes, size_t n);
+void tls_buffer_append_str(struct tls_buffer *b, const unsigned char *bytes);
 void tls_buffer_append16(struct tls_buffer *b, uint16_t n);
 void tls_buffer_append24(struct tls_buffer *b, uint32_t n);
 void tls_buffer_append_byte(struct tls_buffer *b, uint8_t n);
index 8920467058eb3c76f7c57e13c19a2afa8fee3e53..233fe26d9c410e04a9771f888b2b2443c41b9cc0 100644 (file)
@@ -61,6 +61,8 @@
 
 #define CHECK_HANDSHAKE_STATE(context, n, limit)  { if (context->hs_messages[n] >= limit) { DEBUG_PRINT("* UNEXPECTED MESSAGE (%i)\n", (int)n); payload_res = TLS_UNEXPECTED_MESSAGE; break; } context->hs_messages[n]++; }
 
+//#define MARK fprintf(stderr, "%s %s:%d\n", __FILE__, __func__, __LINE__)
+#define MARK
 typedef enum {
        KEA_dhe_dss,
        KEA_dhe_rsa,
@@ -2441,16 +2443,7 @@ int tls_random(unsigned char *key, int len) {
 }
 
 int tls_established(struct TLSContext *context) {
-       if (context) {
-               if (context->critical_error) {
-                       return -1;
-               }
-
-               if (context->connection_status == TLS_CONNECTED) {
-                       return 1;
-               }
-       }
-       return 0;
+       return context && context->connection_status == TLS_CONNECTED;
 }
 
 void tls_read_clear(struct TLSContext *context) {
@@ -5567,21 +5560,35 @@ int tls_connect(struct TLSContext *context) {
        int res;
        ssize_t read_size;
 
-       if (!context || context->fd <= 0 || context->critical_error) {
+       MARK;
+       if (!context || context->fd < 0 || context->critical_error) {
+               if (!context) {
+                       MARK;
+               } else if (context->fd < 0) {
+                       MARK;
+               } else {
+                       MARK;
+               }
+                       
                return TLS_GENERIC_ERROR;
        }
 
+       MARK;
        if (context->is_server) {
                return TLS_UNEXPECTED_MESSAGE;
        }
+       MARK;
 
        res = tls_queue_packet(tls_build_client_hello(context));
 
+       MARK;
        if (res < 0) {
                return res;
        }
 
+       MARK;
        res = tls_fsync(context);
+       MARK;
        if (res < 0) {
                return res;
        }
@@ -5593,13 +5600,18 @@ int tls_connect(struct TLSContext *context) {
                                return res;
                        }
                }
+       MARK;
                if (tls_established(context)) {
                        return 1;
                }
+       MARK;
                if (context->critical_error) {
+                       fprintf(stderr, "critical error: %d\n",
+                                       context->critical_error);
                        return TLS_GENERIC_ERROR;
                }
        }
+       MARK;
        return read_size;
 }
 
@@ -5677,7 +5689,7 @@ ssize_t tls_read(struct TLSContext *context, void *buf, size_t count) {
                return TLS_GENERIC_ERROR;
        }
 
-       if (tls_established(context) != 1) {
+       if (!tls_established(context)) {
                return TLS_GENERIC_ERROR;
        }
 
diff --git a/doc/zpm-fetchurl.8 b/doc/zpm-fetchurl.8
new file mode 100644 (file)
index 0000000..c6f748c
--- /dev/null
@@ -0,0 +1,73 @@
+.TH zpm-fetchurl 8 2018-12-09 "ZPM 0.4"
+.SH NAME
+zpm-fetchurl \- download files
+.SH SYNOPSIS
+.B zpm fetchurl
+[
+.B -ISkKr
+]
+[
+.BI -o file
+]
+.I [ url ]
+.SH DESCRIPTION
+\fBzpm-fetchurl\fR downloads files
+.PP
+\fBzpm-fetchurl\fR understands http, https, and file type urls.
+For https, it uses a trust on first use model for ssl certificates, but a more
+common model verifying via root certificates, or no verfication can also be
+used.
+.PP
+While this program can be used directly, it is intended for use by zpm scripts
+for downloading repositories and packages without requiring an external
+dependency.
+.SH OPTIONS
+.TP
+.B \-I
+output the response header only, implies -r
+.TP
+.B \-r
+output the entire response header, including the header
+.TP
+.B \-S
+output the response status code only
+.TP
+.B \-k
+Don't verify any certificates.
+.TP
+.B \-K
+Verify certificates via root certificates.  This is normally the default for
+TLS, but zpm-fetchurl uses a trust on first use policy instead.  Root
+certificates are found by default in /etc/zpm/roots.pem, but the ZPM_ROOTFILE
+environment variable can be used to override this.
+.TP
+.BI \-R " limit"
+set the limit on following redirects, defaults to 50
+.TP
+.BI \-z " path"
+Only download if remote is newer than the file given by path
+.TP
+.BI \-o " path"
+Write the output to the file given by \fIpath\fR.  Output is
+written to stdout by default.
+.TP
+.B \-f
+Fail silently on errors.
+.TP
+.B \-#
+Output a progress bar.
+.SH EXAMPLES
+.TP
+zpm fetchurl -o zpm-0.1.2.zpm https://zoranix.net/repo/packages/zpm-0.1.2.zpm
+.SH EXIT STATUS
+0 on success non zero on failure
+.SH FILES
+/var/lib/zpm/known_hosts
+~/.zpm/known_hosts
+.SH ENVIRONMENT
+ZPM_KNOWNHOSTS
+ZPM_ROOTFILE
+.SH AUTHOR
+Nathan Wagner
+.SH SEE ALSO
+zpm(8)
index 6fa512e4dbaa690c9f37ef72f131a58bb6c6d80f..cb36c21f0e76580097fc312c9099e1bd8c4dc687 100644 (file)
--- a/doc/zpm.8
+++ b/doc/zpm.8
@@ -99,3 +99,4 @@ the musl copyright.
 .BR zpm-repo (8)
 .BR zpm-quote (8)
 .BR zpm-hash (8)
+.BR zpm-fetchurl (8)
index 4ffca2333b8cd16f994fcddef20fdd4e490073f5..1b337244421df9f0f74b7aa5a2d99186670ecd60 100644 (file)
@@ -15,6 +15,7 @@
 #include <fcntl.h>
 
 #include "tlse.h"
+#define MARK fprintf(stderr, "%s %s:%d\n", __FILE__, __func__, __LINE__)
 
 struct tls_uri {
        char *scheme;
@@ -132,29 +133,38 @@ int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int
                if (geteuid() == 0) {
                        trustfile = "/var/lib/zpm/known_hosts";
                } else {
-                       homedir = getenv("HOME");
-                       if (!homedir) {
+                       /* we could do this with a series of
+                        * openat() calls instead of building
+                        * up a string 
+                        */
+                       trustfile = getenv("HOME");
+                       if (!trustfile) {
+                               fprintf(stderr, "home = %s\n", trustfile);
                                return 1;
                        }
-                       len = snprintf(homedir, 0, "%s/.zpm/known_hosts", homedir);
+                       len = snprintf(homedir, 0, "%s/.zpm/known_hosts", trustfile);
                        homedir = malloc(len+1);
                        if (!homedir) {
                                return 1;
                        }
-                       len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", homedir);
+                       len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", trustfile);
                        trustfile = homedir;
                }
        }
        /* cert is valid on its face, so check against the trust db */
        trustdb = open(trustfile, O_RDWR|O_CREAT, 0600);
-       if (homedir) {
-               free(homedir);
-       }
        if (trustdb == -1) {
-               fprintf(stderr, "cannot open trustdb: %s\n", strerror(errno));
+               fprintf(stderr, "cannot open trustdb %s: %s\n", trustfile, strerror(errno));
+               if (homedir) {
+                       free(homedir);
+               }
                return 1;
        }
 
+       if (homedir) {
+               free(homedir);
+       }
+
        len = 0;
        tls_buffer_init(&tbuf, 128);
        do {
@@ -205,10 +215,17 @@ int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int
 
        /* got here, so we should be at EOF, so add this host to trust db */
        lseek(trustdb, 0, SEEK_END);
-       write(trustdb, certhash, 64);
-       write(trustdb, ":", 1);
-       write(trustdb, context->sni, strlen(context->sni));
-       write(trustdb, "\n", 1);
+
+       /* re-use the buffer so we only do one write */
+       /* ignore errors, the cert is fine, we just can't update
+        * the trustdb if there's errors here
+        */
+       tbuf.len = 0;
+       tls_buffer_append(&tbuf, certhash, 64);
+       tls_buffer_append_byte(&tbuf, ':');
+       tls_buffer_append(&tbuf, context->sni, strlen(context->sni));
+       tls_buffer_append_byte(&tbuf, '\n');
+       write(trustdb, tbuf.buffer, tbuf.len);
        close(trustdb);
        tls_buffer_free(&tbuf);
 
@@ -222,17 +239,13 @@ int verify_roots(struct TLSContext *context, struct TLSCertificate **chain, int
        if (chain) {
                for (i = 0; i < len; i++) {
                        struct TLSCertificate *certificate = chain[i];
-                       // check validity date
                        err = tls_certificate_is_valid(certificate);
                        if (err) {
                                return err;
                        }
-                       // check certificate in certificate->bytes of length
-                       // certificate->len the certificate is in ASN.1 DER
-                       // format
                }
        }
-       // check if chain is valid
+       
        err = tls_certificate_chain_is_valid(chain, len);
        if (err) {
                return err;
@@ -280,11 +293,9 @@ int month(char *m) {
 
 /* Wed, 06 Feb 2019 10:06:05 GMT */
 time_t parse_date(char *d) {
-       //static const char format[] = "%a, %d %b %Y %H:%M:%S %Z"; // rfc 1123
        struct tm tm = { 0 };
        int rv;
 
-       //char *data = "Tue, 13 Dec 2011 16:08:21 GMT";
        int h, m, s, dom, Y;
        char M[4];
        rv = sscanf(d, "%*[a-zA-Z,] %d %s %d %d:%d:%d", &dom, M, &Y, &h, &m, &s);
@@ -425,6 +436,16 @@ void append_header(struct tls_buffer *buf, char *header, char *val) {
        tls_buffer_append(buf, "\r\n", 2);
 }
 
+void append_timeheader(struct tls_buffer *buf, char *header, time_t ts) {
+       char timestr[80];
+       struct tm *tm;
+
+       tm = gmtime(&ts);
+
+       strftime(timestr, sizeof timestr, "%a, %d %b %Y %H:%M:%S GMT", tm);
+       append_header(buf, header, timestr);
+}
+
 static void pdots(int len, int ch, unsigned long was, unsigned long now,
                unsigned long total) {
        was = len * was / total;
@@ -437,6 +458,56 @@ static void pdots(int len, int ch, unsigned long was, unsigned long now,
        }
 }
 
+static void fake_header(struct io *io, int fd) {
+       struct stat st;
+       int code = 200, rv;
+       char *message, codestr[5], length[32];
+       struct tls_buffer *hdr = &io->response;
+
+       if (fd == -1) {
+               switch (errno) {
+                       case EACCES: code = 403; break;
+                       case ENOENT: code = 404; break;
+                       default: code = 500; break;
+               }
+       } else {
+               rv = fstat(fd, &st);
+               if (rv == -1) {
+                       code = 500;
+               }
+       }
+
+       if (io->last_modified >= st.st_mtime) {
+               code = 304;
+       }
+
+       switch (code) {
+               case 200: message = "OK"; break;
+               case 304: message = "Not Modified"; break;
+               case 403: message = "Forbidden"; break;
+               case 404: message = "Not Found"; break;
+               case 500: message = "Internal Server Error"; break;
+               default: break;
+       }
+       sprintf(codestr, "%0.3d ", code);
+       tls_buffer_append(hdr, "HTTP/1.1 ", 9);
+       tls_buffer_append(hdr, codestr, 4);
+       tls_buffer_append_str(hdr, message);
+       tls_buffer_append(hdr, "\r\n", 2);
+
+       append_timeheader(hdr, "Date", time(NULL));
+       append_header(hdr, "Server", "zpm-fetchurl/0.9");
+       if (code < 400) {
+               append_timeheader(hdr, "Last-Modified", st.st_mtime);
+               sprintf(length, "%zu", st.st_size);
+               append_header(hdr, "Content-Length", length);
+       }
+
+       append_header(hdr, "Connection", "close");
+       append_header(hdr, "Content-Type", "application/octet-stream");
+       tls_buffer_append(hdr, "\r\n", 2);
+}
+
 int main(int ac, char *av[]) {
        int sockfd, port = -1, rv;
        ssize_t ret;
@@ -492,6 +563,7 @@ int main(int ac, char *av[]) {
                exit(EXIT_FAILURE);
        }
 
+       io.last_modified = 0;
        if (lmfile) {
                struct stat st;
                int rv;
@@ -504,6 +576,7 @@ int main(int ac, char *av[]) {
                        exit(EXIT_FAILURE);
                }
                ts = st.st_mtime;
+               io.last_modified = ts;
                mtime = gmtime(&ts);
                strftime(lmtime, sizeof lmtime, "%a, %d %b %Y %H:%M:%S GMT", mtime);
        }
@@ -543,8 +616,7 @@ int main(int ac, char *av[]) {
                        tls_buffer_append(&request, "GET ", 4);
                }
                tls_buffer_append(&request, uri.path, strlen(uri.path));
-               tls_buffer_append(&request, " HTTP/1.1", 9);
-               tls_buffer_append(&request, "\r\n", 2);
+               tls_buffer_append(&request, " HTTP/1.1\r\n", 11);
 
                append_header(&request, "Host", host);
                append_header(&request, "Connection", "close");
@@ -552,7 +624,6 @@ int main(int ac, char *av[]) {
                        append_header(&request, "If-Modified-Since", lmtime);
                }
                tls_buffer_append(&request, "\r\n", 2);
-               //fprintf(stderr, "msg =\n%.*s", (int)request.len, request.buffer);
 
                if (!strcmp(uri.scheme, "https")) {
                        use_tls = 1;
@@ -589,41 +660,49 @@ int main(int ac, char *av[]) {
                        tls_sni_set(clientssl, uri.host);
                        clientssl->sync = 1;
                        io.tls = clientssl;
-               }
-
-               sockfd = open_tcp_connection(host, port);
-               io.socket = sockfd;
-
-               if (sockfd < 0) {
-                       exit(EXIT_FAILURE);
-               }
-
-               if (use_tls) {
+                       sockfd = open_tcp_connection(host, port);
+                       if (sockfd < 0) {
+                               perror("can't open connection");
+                               exit(EXIT_FAILURE);
+                       }
                        tls_set_fd(clientssl, sockfd);
-
                        if ((rv = tls_connect(clientssl)) != 1) {
                                fprintf(stderr, "Handshake Error %i\n", rv);
-                               return -5;
+                               return 1;
                        }
-
                        ret = tls_write(clientssl, request.buffer, request.len);
+               } else if (!strcmp(uri.scheme, "http")) {
+                       sockfd = open_tcp_connection(host, port);
+                       if (sockfd < 0) {
+                               perror("can't open connection");
+                               exit(EXIT_FAILURE);
+                       }
+                       ret = write(sockfd, request.buffer, request.len);
+               } else if (!strcmp(uri.scheme, "file")) {
+                       sockfd = open(uri.path, O_RDONLY);
+                       fake_header(&io, sockfd);
+                       ret = 0;
                } else {
-                       ret = write(io.socket, request.buffer, request.len);
+                       fprintf(stderr, "scheme %s unknown\n", uri.scheme);
+                       exit(EXIT_FAILURE);
                }
 
-               if (ret < 0) {
-                       fprintf(stderr, "write error %zd\n", ret);
-                       return -6;
+               if (ret == -1) {
+                       fprintf(stderr, "unable to write http request: %s\n", strerror(errno));
+                       exit(EXIT_FAILURE);
                }
 
+               io.socket = sockfd;
+
                do {
-                       ret = fill_buffer(&io);
-                       if (ret < 0) {
-                               break;
+                       if (io.response.len >= 4) {
+                               eoh = strstr(io.response.buffer, "\r\n\r\n");
                        }
-                       eoh = strstr(io.response.buffer, "\r\n\r\n");
-                       if (ret == 0) {
-                               break;
+                       if (!eoh) {
+                               ret = fill_buffer(&io);
+                               if (ret <= 0) {
+                                       break;
+                               }
                        }
                } while (!eoh);
 
@@ -652,10 +731,6 @@ int main(int ac, char *av[]) {
                        break;
                }
 
-               if (io.status_code != 200) {
-                       break;
-               }
-
                if (!raw) {
                        tls_buffer_shift(&io.response, header_len);
                }
@@ -687,6 +762,9 @@ int main(int ac, char *av[]) {
                                }
                                total += ret;
                        }
+                       if (head) {
+                               break;
+                       }
                        ret = fill_buffer(&io);
                } while (ret > 0);