pd.if.org Git - zpackage/blob - src/fetchurl.c

   1 #define _POSIX_C_SOURCE 200809L
   2
   3 #include <stdio.h>
   4 #include <sys/types.h>
   5 #include <sys/socket.h>
   6 #include <netinet/in.h>
   7 #include <netdb.h>
   8 #include <string.h>
   9 #include <strings.h>
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <time.h>
  14 #include <sys/stat.h>
  15 #include <fcntl.h>
  16
  17 #include "tlse.h"
  18
  19 struct tls_uri {
  20         char *scheme;
  21         char *userinfo;
  22         char *host;
  23         char *port;
  24         char *path;
  25         char *encoded_path;
  26         char *query;
  27         char *encoded_query;
  28         char *fragment;
  29 };
  30
  31 int tls_parse_uri(char *, struct tls_uri *);
  32 void tls_free_uri(struct tls_uri *);
  33
  34 int open_tcp_connection(char *host, int port);
  35
  36 /* if trustpolicy is 0, we just accept anything */
  37 int verify_trust(struct TLSContext *context, struct TLSCertificate **chain, int
  38                 len) {
  39         /* suppress unused */
  40         if (context || chain || len) {
  41                 return 0;
  42         }
  43
  44         return 0;
  45 }
  46
  47 static char hexchars[] = "0123456789abcdefABCDEF";
  48
  49 static void hex(char *dst, uint8_t *src, size_t len) {
  50         while (len--) {
  51                 dst[0] = hexchars[(src[0]>>4)&0xf];
  52                 dst[1] = hexchars[src[0]&0xf];
  53                 dst+=2;
  54                 src++;
  55         }
  56 }
  57
  58 #if 0
  59 static void hexbin(uint8_t *dst, unsigned char *src, size_t len) {
  60         size_t i;
  61         int x;
  62
  63         for (i=0; i<len; i+=2) {
  64                 sscanf((const char *)src+i, "%02x", &x);
  65                 dst[i/2] = x;
  66         }
  67 }
  68 #endif
  69
  70 char *my_getline(struct tls_buffer *b, int fd, size_t *size) {
  71         char *loc = 0;
  72         char buf[4096];
  73         ssize_t rv = 0;
  74
  75         while (b->error == 0) {
  76                 loc = memchr(b->buffer, '\n', b->len);
  77                 if (loc) {
  78                         *size = loc - b->buffer + 1;
  79                         return b->buffer;
  80                 } else {
  81                         rv = read(fd, buf, sizeof buf);
  82                         if (rv == -1) {
  83                                 return 0;
  84                         }
  85                         if (rv == 0) {
  86                                 break;
  87                         }
  88                         tls_buffer_append(b, buf, rv);
  89                 }
  90         }
  91         if (rv == 0) {
  92                 *size = b->len;
  93                 return b->buffer;
  94         }
  95         return 0;
  96 }
  97
  98 /*
  99  * We use a trust on first use policy.  The trust DB is a simple
 100  * file in /var/lib/zpm/known_hosts, or ~/.zpm/known_hosts, or ZPM_KNOWNHOSTS.
 101  * if -k is given, no verification is done
 102  */
 103 int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int
 104                 certs) {
 105         int err;
 106         char *trustfile, *homedir = 0, *host, *fp;
 107         unsigned char certhash[65];
 108         int trustdb;
 109         struct tls_buffer tbuf;
 110
 111         char *line = 0;
 112         size_t len = 0;
 113
 114         if (certs == 0 || chain == 0) {
 115                 return 1;
 116         }
 117
 118         err = tls_certificate_is_valid(chain[0]);
 119         if (err) {
 120                 return err;
 121         }
 122
 123         if (context->sni) {
 124                 err = tls_certificate_valid_subject(chain[0], context->sni);
 125                 if (err) {
 126                         return err;
 127                 }
 128         }
 129
 130         hex(certhash, chain[0]->fp, 32);
 131         certhash[64] = 0;
 132
 133         trustfile = getenv("ZPM_KNOWNHOSTS");
 134         if (!trustfile) {
 135                 if (geteuid() == 0) {
 136                         trustfile = "/var/lib/zpm/known_hosts";
 137                 } else {
 138                         /* we could do this with a series of
 139                          * openat() calls instead of building
 140                          * up a string
 141                          */
 142                         trustfile = getenv("HOME");
 143                         if (!trustfile) {
 144                                 fprintf(stderr, "home = %s\n", trustfile);
 145                                 return 1;
 146                         }
 147                         len = snprintf(homedir, 0, "%s/.zpm/known_hosts", trustfile);
 148                         homedir = malloc(len+1);
 149                         if (!homedir) {
 150                                 return 1;
 151                         }
 152                         len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", trustfile);
 153                         trustfile = homedir;
 154                 }
 155         }
 156         /* cert is valid on its face, so check against the trust db */
 157         trustdb = open(trustfile, O_RDWR|O_CREAT, 0600);
 158         if (trustdb == -1) {
 159                 fprintf(stderr, "cannot open trustdb %s: %s\n", trustfile, strerror(errno));
 160                 if (homedir) {
 161                         free(homedir);
 162                 }
 163                 return 1;
 164         }
 165
 166         if (homedir) {
 167                 free(homedir);
 168         }
 169
 170         len = 0;
 171         tls_buffer_init(&tbuf, 128);
 172         do {
 173                 char *off;
 174
 175                 tls_buffer_shift(&tbuf, len);
 176                 line = my_getline(&tbuf, trustdb, &len);
 177
 178                 if (!line || !len) {
 179                         break;
 180                 }
 181
 182                 fp = line;
 183                 while (isspace(*fp)) {
 184                         fp++;
 185                 }
 186                 if (*fp == '#') {
 187                         continue;
 188                 }
 189                 off = strchr(line, ':');
 190                 if (!off) {
 191                         continue;
 192                 }
 193                 host = off + 1;
 194                 *off = 0;
 195                 if (line[len-1] == '\n') {
 196                         line[len-1] = 0;
 197                 }
 198
 199                 if (strlen(fp) != 64) {
 200                         continue;
 201                 }
 202
 203                 if (len && line[len-1] == '\n') {
 204                         line[len-1] = 0;
 205                 }
 206
 207                 if (strcmp(context->sni, host) != 0) {
 208                         continue;
 209                 }
 210
 211                 int match = (memcmp(certhash, fp, 64) == 0);
 212                 if (!match) {
 213                         fprintf(stderr, "host %s certificate changed\n", host);
 214                         fprintf(stderr, "was %.64s\n", fp);
 215                         fprintf(stderr, "now %.64s\n", certhash);
 216                 }
 217
 218                 close(trustdb);
 219                 tls_buffer_free(&tbuf);
 220                 return match ? no_error : bad_certificate;
 221         } while (!tbuf.error);
 222
 223         /* got here, so we should be at EOF, so add this host to trust db */
 224         lseek(trustdb, 0, SEEK_END);
 225
 226         /* re-use the buffer so we only do one write */
 227         /* ignore errors, the cert is fine, we just can't update
 228          * the trustdb if there's errors here
 229          */
 230         tbuf.len = 0;
 231         tls_buffer_append(&tbuf, certhash, 64);
 232         tls_buffer_append_byte(&tbuf, ':');
 233         tls_buffer_append(&tbuf, context->sni, strlen(context->sni));
 234         tls_buffer_append_byte(&tbuf, '\n');
 235         write(trustdb, tbuf.buffer, tbuf.len);
 236         close(trustdb);
 237         tls_buffer_free(&tbuf);
 238
 239         return no_error;
 240 }
 241
 242 int verify_roots(struct TLSContext *context, struct TLSCertificate **chain, int
 243                 len) {
 244         int i, err;
 245
 246         if (chain) {
 247                 for (i = 0; i < len; i++) {
 248                         struct TLSCertificate *certificate = chain[i];
 249                         err = tls_certificate_is_valid(certificate);
 250                         if (err) {
 251                                 return err;
 252                         }
 253                 }
 254         }
 255
 256         err = tls_certificate_chain_is_valid(chain, len);
 257         if (err) {
 258                 return err;
 259         }
 260
 261         if (len > 0 && context->sni) {
 262                 err = tls_certificate_valid_subject(chain[0], context->sni);
 263                 if (err) {
 264                         return err;
 265                 }
 266         }
 267
 268         /* Perform certificate validation against ROOT CA */
 269         err = tls_certificate_chain_is_valid_root(context, chain, len);
 270         if (err) {
 271                 return err;
 272         }
 273
 274         return no_error;
 275 }
 276
 277 struct io {
 278         struct tls_buffer response;
 279         struct tls_buffer chunkbuf;
 280         struct TLSContext *tls;
 281         int socket;
 282         int chunked;
 283         int chunknum;
 284         size_t chunksize;
 285         size_t chunkleft;
 286         size_t chunktotal;
 287         size_t chunkbytesread;
 288         int status_code;
 289         time_t last_modified;
 290         time_t date;
 291         size_t content_length;
 292         size_t received;
 293         char *redirect;
 294 };
 295 ssize_t unchunk(struct io *io);
 296
 297 int month(char *m) {
 298         char *months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul",
 299                 "Aug", "Sep", "Oct", "Nov", "Dec" };
 300         int i;
 301
 302         for (i=0; i < 12; i++) {
 303                 if (!strncasecmp(m, months[i], 3)) {
 304                         return i+1;
 305                 }
 306         }
 307         return 0;
 308 }
 309
 310 /* Wed, 06 Feb 2019 10:06:05 GMT */
 311 time_t parse_date(char *d) {
 312         struct tm tm = { 0 };
 313         int rv;
 314
 315         int h, m, s, dom, Y;
 316         char M[4];
 317         rv = sscanf(d, "%*[a-zA-Z,] %d %s %d %d:%d:%d", &dom, M, &Y, &h, &m, &s);
 318         if (rv == 6) {
 319                 tm.tm_year = Y - 1900;
 320                 tm.tm_hour = h;
 321                 tm.tm_min = m;
 322                 tm.tm_sec = s;
 323                 tm.tm_mon = month(M)-1;
 324                 tm.tm_mday = dom;
 325
 326                 return mktime(&tm);
 327         }
 328         return -1;
 329 }
 330
 331 char *find_header(struct io *io, char *header, size_t *len) {
 332         char *eoh, *soh;
 333         size_t hlen;
 334
 335         *len = 0;
 336
 337         hlen = strlen(header);
 338         eoh = strstr(io->response.buffer, "\r\n\r\n");
 339         if (!eoh) {
 340                 return 0;
 341         }
 342         soh = io->response.buffer;
 343         do {
 344                 soh = strstr(soh, "\r\n");
 345                 if (soh == eoh) {
 346                         break;
 347                 }
 348                 soh += 2;
 349                 if (!memcmp(soh, header, hlen)) {
 350                         break;
 351                 }
 352         } while (soh < eoh);
 353
 354         if (soh >= eoh) {
 355                 return 0;
 356         }
 357         eoh = strstr(soh, "\r\n");
 358         soh += hlen;
 359         while (soh < eoh && isspace(*soh)) {
 360                 soh++;
 361         }
 362         *len = (eoh - soh);
 363         return soh;
 364 }
 365
 366 void parse_header(struct io *io) {
 367         char *s = io->response.buffer;
 368         int code = 0;
 369         char *hval;
 370         size_t hlen;
 371
 372         while (!isspace(*s)) {
 373                 s++;
 374         }
 375         while (isspace(*s)) {
 376                 s++;
 377         }
 378         code = strtol(s, 0, 10);
 379         io->status_code = code;
 380
 381         hval = find_header(io, "Date:", &hlen);
 382         if (hval) {
 383                 hval[hlen] = 0;
 384                 io->date = parse_date(hval);
 385                 hval[hlen] = '\r';
 386         }
 387         hval = find_header(io, "Last-Modified:", &hlen);
 388         if (hval) {
 389                 hval[hlen] = 0;
 390                 io->last_modified = parse_date(hval);
 391                 hval[hlen] = '\r';
 392         }
 393
 394         hval = find_header(io, "Content-Length:", &hlen);
 395         if (hval) {
 396                 hval[hlen] = 0;
 397                 io->content_length = strtoul(hval, 0, 10);
 398                 hval[hlen] = '\r';
 399         }
 400
 401         hval = find_header(io, "Transfer-Encoding:", &hlen);
 402         if (hval) {
 403                 hval[hlen] = 0;
 404                 io->content_length = strtoul(hval, 0, 10);
 405                 if (!strcmp(hval, "chunked")) {
 406                         io->chunked = 1;
 407                 }
 408                 hval[hlen] = '\r';
 409         }
 410
 411         switch (code) {
 412                 case 301:
 413                 case 302:
 414                 case 303:
 415                 case 307:
 416                         hval = find_header(io, "Location:", &hlen);
 417                         if (hval) {
 418                                 io->redirect = strndup(hval, hlen);
 419                         }
 420                         break;
 421                 default:
 422                         break;
 423         }
 424
 425 }
 426
 427 /* fill buffer needs to put bytes into the response buffer
 428  * if the transfer encoding is chunked, it will need to
 429  * put the bytes into the chunkbuf first, then call
 430  * unchunk.  if unchunk return 0, then it needs more data,
 431  * otherwise unchunk returns the number of bytes transferred
 432  */
 433
 434 ssize_t fill_buffer(struct io *io) {
 435         unsigned char buffer[4096];
 436         ssize_t ret = 0;
 437
 438         ret = unchunk(io);
 439
 440         while (ret == 0) {
 441                 if (io->tls) {
 442                         ret = tls_read(io->tls, buffer, sizeof buffer);
 443                 } else {
 444                         ret = read(io->socket, buffer, sizeof buffer);
 445                 }
 446
 447                 if (ret <= 0) {
 448                         break;
 449                 }
 450
 451                 if (io->chunked) {
 452                         tls_buffer_append(&io->chunkbuf, buffer, ret);
 453                         //fwrite(buffer, ret, 1, stderr);
 454                         ret = unchunk(io);
 455                         if (ret != 0 || io->chunksize == 0) {
 456                                 break;
 457                         }
 458                 } else {
 459                         tls_buffer_append(&io->response, buffer, ret);
 460                         break;
 461                 }
 462         }
 463
 464         fprintf(stderr, "filled %zd bytes\n", ret);
 465         return ret;
 466 }
 467
 468 /* essentially memmem */
 469 void *lookfor(const void *buf, size_t buflen, const void *pattern, size_t len) {
 470       const char *bf = buf;
 471       const char *pt = pattern;
 472       const char *p = bf;
 473
 474       while (len <= (buflen - (p - bf))) {
 475             if ((p = memchr(p, *pt, buflen - (p - bf))) != 0) {
 476                   if (memcmp(p, pattern, len) == 0) {
 477                         return (void *)p;
 478                   } else {
 479                           p++;
 480                   }
 481             } else {
 482                     break;
 483             }
 484       }
 485       return NULL;
 486 }
 487
 488 /* returns read chunksize, unshifts the line */
 489 ssize_t read_chunksize(struct io *io) {
 490         char *cr;
 491         ssize_t cs;
 492
 493         //fwrite(io->chunkbuf.buffer, io->chunkbuf.len, 1, stderr);
 494
 495         /* there could be up to two leading bytes */
 496         if (io->chunkbuf.len >= 2 && io->chunkbuf.buffer[0] == '\r' && io->chunkbuf.buffer[1] == '\n') {
 497                 tls_buffer_shift(&io->chunkbuf, 2);
 498         }
 499
 500         cr = lookfor(io->chunkbuf.buffer, io->chunkbuf.len, "\r\n", 2);
 501
 502         if (cr == 0) {
 503                 return -1;
 504         }
 505
 506         cs = strtol(io->chunkbuf.buffer, 0, 16);
 507         tls_buffer_shift(&io->chunkbuf, cr - io->chunkbuf.buffer + 2);
 508
 509         return cs;
 510 }
 511
 512 /* unchunk's job is to move bytes from the chunk buf to the response buf */
 513 /* return bytes from chunk, 0 if unable.  once last chunk, changed chunked
 514  * to 0?
 515  */
 516 ssize_t unchunk(struct io *io) {
 517         ssize_t bytes_to_move = 0;
 518         ssize_t chunksize;
 519
 520         if (!io || !io->chunked) {
 521                 return 0;
 522         }
 523
 524         if (io->chunkleft == 0) {
 525                 chunksize = read_chunksize(io);
 526                 if (chunksize == -1) {
 527                         return 0;
 528                 }
 529                 io->chunksize = chunksize;
 530                 if (io->chunksize == 0) {
 531                         /* end of chunked data */
 532                         io->chunked = 0;
 533                         return 0;
 534                 }
 535                 io->chunknum++;
 536                 io->chunkleft = io->chunksize;
 537                 io->chunktotal += io->chunksize;
 538         }
 539
 540         if (io->chunkbuf.len == 0) {
 541                 /* need more bytes */
 542                 return 0;
 543         }
 544
 545         bytes_to_move = io->chunkbuf.len < io->chunkleft ? io->chunkbuf.len : io->chunkleft;
 546
 547         tls_buffer_append(&io->response, io->chunkbuf.buffer, bytes_to_move);
 548         io->chunkleft -= bytes_to_move;
 549         io->chunkbytesread += bytes_to_move;
 550
 551         /* chunk is terminated with a crlf */
 552         //tls_buffer_shift(&io->chunkbuf, bytes_to_move + io->chunkleft ? 0 : 2);
 553         tls_buffer_shift(&io->chunkbuf, bytes_to_move);
 554
 555         return bytes_to_move;
 556 }
 557
 558 #if 0
 559 char *nextline(struct io *io) {
 560         char *eol = 0;;
 561
 562         eol = memchr(io->response.buffer, '\n', io.response.size);
 563         while (eol == 0) {
 564                 fill_buffer(io);
 565                 eol = memchr(io->response.buffer, '\n', io.response.size);
 566         }
 567         if (eol) {
 568
 569 }
 570 #endif
 571
 572 void append_header(struct tls_buffer *buf, char *header, char *val) {
 573         tls_buffer_append(buf, header, strlen(header));
 574         tls_buffer_append(buf, ": ", 2);
 575         tls_buffer_append(buf, val, strlen(val));
 576         tls_buffer_append(buf, "\r\n", 2);
 577 }
 578
 579 void append_timeheader(struct tls_buffer *buf, char *header, time_t ts) {
 580         char timestr[80];
 581         struct tm *tm;
 582
 583         tm = gmtime(&ts);
 584
 585         strftime(timestr, sizeof timestr, "%a, %d %b %Y %H:%M:%S GMT", tm);
 586         append_header(buf, header, timestr);
 587 }
 588
 589 static void pdots(int len, int ch, unsigned long was, unsigned long now,
 590                 unsigned long total) {
 591         was = len * was / total;
 592         if (now > total) {
 593                 now = total;
 594         }
 595         now = len * now / total;
 596         while (was++ < now) {
 597                 putc(ch,stderr);
 598         }
 599 }
 600
 601 static void fake_header(struct io *io, int fd) {
 602         struct stat st;
 603         int code = 200, rv;
 604         char *message, codestr[5], length[32];
 605         struct tls_buffer *hdr = &io->response;
 606
 607         if (fd == -1) {
 608                 switch (errno) {
 609                         case EACCES: code = 403; break;
 610                         case ENOENT: code = 404; break;
 611                         default: code = 500; break;
 612                 }
 613         } else {
 614                 rv = fstat(fd, &st);
 615                 if (rv == -1) {
 616                         code = 500;
 617                 }
 618         }
 619
 620         if (io->last_modified >= st.st_mtime) {
 621                 code = 304;
 622         }
 623
 624         switch (code) {
 625                 case 200: message = "OK"; break;
 626                 case 304: message = "Not Modified"; break;
 627                 case 403: message = "Forbidden"; break;
 628                 case 404: message = "Not Found"; break;
 629                 case 500: message = "Internal Server Error"; break;
 630                 default: break;
 631         }
 632         sprintf(codestr, "%0.3d ", code);
 633         tls_buffer_append(hdr, "HTTP/1.1 ", 9);
 634         tls_buffer_append(hdr, codestr, 4);
 635         tls_buffer_append_str(hdr, message);
 636         tls_buffer_append(hdr, "\r\n", 2);
 637
 638         append_timeheader(hdr, "Date", time(NULL));
 639         append_header(hdr, "Server", "zpm-fetchurl/0.9");
 640         if (code < 400) {
 641                 append_timeheader(hdr, "Last-Modified", st.st_mtime);
 642                 sprintf(length, "%zu", st.st_size);
 643                 append_header(hdr, "Content-Length", length);
 644         }
 645
 646         append_header(hdr, "Connection", "close");
 647         append_header(hdr, "Content-Type", "application/octet-stream");
 648         tls_buffer_append(hdr, "\r\n", 2);
 649 }
 650
 651 char *pathlast(char *path) {
 652         char *last = 0;
 653         size_t len = 0;
 654
 655         if (path == 0) {
 656                 return 0;
 657         }
 658
 659         while (*path == '/') {
 660                 path++;
 661         }
 662
 663         do {
 664                 last = path;
 665                 len = 0;
 666                 while (*path && *path != '/') {
 667                         path++;
 668                         len++;
 669                 }
 670                 while (*path == '/') {
 671                         path++;
 672                 }
 673         } while (*path);
 674
 675         if (len == 0) {
 676                 return 0;
 677         }
 678
 679         return strndup(last, len);
 680 }
 681
 682 static time_t file_mtime(char *path) {
 683         struct stat st;
 684         int rv;
 685
 686         rv = stat(path, &st);
 687         if (rv == -1) {
 688                 if (errno == ENOENT) {
 689                         return 0;
 690                 }
 691                 perror("stat failed:");
 692                 return -1;
 693         }
 694
 695         return st.st_mtime;
 696 }
 697
 698 int main(int ac, char *av[]) {
 699         int sockfd, port = -1, rv;
 700         ssize_t ret;
 701         int option;
 702 #if 0
 703         char msg[] = "GET %s HTTP/1.1\r\nHost: %s:%i\r\nConnection: close\r\n\r\n";
 704         char msg2[] = "GET %s HTTP/1.1\r\nHost: %s:%i\r\nLast-Modified: %s\r\nConnection: close\r\n\r\n";
 705         char msg_buffer[1024];
 706 #endif
 707         char *req_file = 0;
 708         char *host = 0;
 709         struct tls_uri uri = { 0 };
 710         char *outfile = 0;
 711         int raw = 0, head = 0;
 712         int out = 1; /* output file descriptor */
 713         int use_tls = 0;
 714         struct io io = { {0}, {0}, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 715         struct TLSContext *clientssl = 0;
 716         int failsilent = 0;
 717         char *lmfile = 0;
 718         int progressbar = 0;
 719         struct tls_buffer request;
 720         char lmtime[80];
 721         char *eoh = 0;
 722         char *user_agent = 0;
 723         size_t total = 0;
 724         size_t header_len;
 725         char *url = 0;
 726         int redirs = 0, redirlimit = 50, printstatus = 0, showreq = 0;
 727         int verifypolicy = 1, calcoutfile = 0, ifnewer = 0;
 728
 729         ltc_mp = tfm_desc;
 730
 731         while ((option = getopt(ac, av, "o:OrIfz:np#RL:SkKU:")) != -1) {
 732                 switch (option) {
 733                         case 'o': outfile = optarg; break;
 734                         case 'O': calcoutfile = 1; break;
 735                         case 'S': printstatus = 1; head = 1; break;
 736                         case 'k': verifypolicy = 0; break;
 737                         case 'K': verifypolicy = 2; break;
 738                         case 'U': user_agent = optarg; break;
 739                         case 'I': head = 1;
 740                         case 'r': raw = 1; break;
 741                         case 'R': showreq = 1; break;
 742                         case 'f': failsilent = 1; break;
 743                         case 'z': lmfile = optarg; break;
 744                         case 'n': ifnewer = 1; break;
 745                         case 'L': redirlimit = strtol(optarg, 0, 10); break;
 746                         case 'p':
 747                         case '#': progressbar = 1; break;
 748                         default:
 749                                   exit(EXIT_FAILURE);
 750                                   break;
 751                 }
 752         }
 753
 754         if (ac < optind) {
 755                 fprintf(stderr, "Usage: %s uri\n", av[0]);
 756                 exit(EXIT_FAILURE);
 757         }
 758
 759         url = strdup(av[optind]);
 760         if (!url) {
 761                 exit(EXIT_FAILURE);
 762         }
 763
 764         io.last_modified = 0;
 765
 766         if (calcoutfile && !outfile) {
 767                 tls_parse_uri(url, &uri);
 768                 outfile = pathlast(uri.path);
 769                 /* outfile leaks memory here, so if this
 770                  * were turned into a library function,
 771                  * we'd need to track it
 772                  */
 773                 if (!outfile) {
 774                         fprintf(stderr, "unable to determine outfile\n");
 775                         exit(EXIT_FAILURE);
 776                 }
 777         }
 778
 779         if (ifnewer && outfile && !lmfile) {
 780                 lmfile = outfile;
 781         }
 782
 783         if (lmfile) {
 784                 struct tm *mtime;
 785                 time_t ts;
 786
 787                 ts = file_mtime(lmfile);
 788
 789                 if (ts == -1) {
 790                         exit(EXIT_FAILURE);
 791                 } else if (ts != 0) {
 792                         io.last_modified = ts;
 793                         mtime = gmtime(&ts);
 794                         strftime(lmtime, sizeof lmtime, "%a, %d %b %Y %H:%M:%S GMT", mtime);
 795                 } else {
 796                         lmfile = 0;
 797                 }
 798         }
 799
 800         signal(SIGPIPE, SIG_IGN);
 801
 802         tls_buffer_init(&io.response, 0);
 803         tls_buffer_init(&request, 128);
 804
 805         while (redirs++ <= redirlimit) {
 806                 tls_free_uri(&uri);
 807                 io.response.len = 0;
 808                 io.chunked = 0;
 809                 request.len = 0;
 810                 eoh = 0;
 811
 812                 tls_parse_uri(url, &uri);
 813                 host = uri.host;
 814                 port = atoi(uri.port);
 815                 req_file = uri.path;
 816
 817                 /* construct request */
 818                 if (head) {
 819                         tls_buffer_append(&request, "HEAD ", 5);
 820                 } else {
 821                         tls_buffer_append(&request, "GET ", 4);
 822                 }
 823                 tls_buffer_append(&request, uri.encoded_path, strlen(uri.encoded_path));
 824                 if (uri.encoded_query) {
 825                         tls_buffer_append(&request, "?", 1);
 826                         tls_buffer_append(&request, uri.encoded_query, strlen(uri.encoded_query));
 827                 }
 828                 tls_buffer_append(&request, " HTTP/1.1\r\n", 11);
 829
 830                 append_header(&request, "Host", host);
 831                 if (user_agent) {
 832                         append_header(&request, "User-Agent", user_agent);
 833                 }
 834                 append_header(&request, "Accept", "*/*");
 835                 //append_header(&request, "Accept-Encoding", "chunked, identity;q=0.5");
 836                 append_header(&request, "Connection", "close");
 837                 if (lmfile) {
 838                         append_header(&request, "If-Modified-Since", lmtime);
 839                 }
 840                 tls_buffer_append(&request, "\r\n", 2);
 841
 842                 if (!strcmp(uri.scheme, "https")) {
 843                         use_tls = 1;
 844
 845                         clientssl = tls_create_context(TLS_CLIENT, TLS_V12);
 846
 847                         /* optionally, we can set a certificate validation
 848                          * callback function if set_verify is not called, and
 849                          * root ca is set, `tls_default_verify` will be used
 850                          * (does exactly what `verify` does in this example)
 851                          */
 852                         if (verifypolicy == 2) {
 853                                 char *cert_path = 0;
 854                                 cert_path = getenv("ZPM_CERTFILE");
 855                                 if (!cert_path) {
 856                                         cert_path = "/var/lib/zpm/roots.pem";
 857                                 }
 858                                 rv = tls_load_root_file(clientssl, cert_path);
 859                                 if (rv == -1) {
 860                                         fprintf(stderr, "Error loading root certs\n");
 861                                         return 1;
 862                                 }
 863                                 tls_set_verify(clientssl, verify_roots);
 864                         } else if (verifypolicy == 1) {
 865                                 tls_set_verify(clientssl, verify_first);
 866                         } else {
 867                                 tls_set_verify(clientssl, verify_trust);
 868                         }
 869
 870                         if (!clientssl) {
 871                                 fprintf(stderr, "Error initializing client context\n");
 872                                 return -1;
 873                         }
 874                         tls_sni_set(clientssl, uri.host);
 875                         clientssl->sync = 1;
 876                         io.tls = clientssl;
 877                         sockfd = open_tcp_connection(host, port);
 878                         if (sockfd < 0) {
 879                                 perror("can't open connection");
 880                                 exit(EXIT_FAILURE);
 881                         }
 882                         tls_set_fd(clientssl, sockfd);
 883                         if ((rv = tls_connect(clientssl)) != 1) {
 884                                 fprintf(stderr, "Handshake Error %i\n", rv);
 885                                 return 1;
 886                         }
 887                         ret = tls_write(clientssl, request.buffer, request.len);
 888                 } else if (!strcmp(uri.scheme, "http")) {
 889                         sockfd = open_tcp_connection(host, port);
 890                         if (sockfd < 0) {
 891                                 perror("can't open connection");
 892                                 exit(EXIT_FAILURE);
 893                         }
 894                         ret = write(sockfd, request.buffer, request.len);
 895                 } else if (!strcmp(uri.scheme, "file")) {
 896                         sockfd = open(uri.path, O_RDONLY);
 897                         fake_header(&io, sockfd);
 898                         ret = 0;
 899                 } else {
 900                         fprintf(stderr, "scheme %s unknown\n", uri.scheme);
 901                         exit(EXIT_FAILURE);
 902                 }
 903
 904                 if (ret == -1) {
 905                         fprintf(stderr, "unable to write http request: %s\n", strerror(errno));
 906                         exit(EXIT_FAILURE);
 907                 }
 908
 909                 io.socket = sockfd;
 910
 911                 eoh = 0;
 912                 do {
 913                         if (io.response.len >= 4) {
 914                                 eoh = strstr(io.response.buffer, "\r\n\r\n");
 915                         }
 916                         if (!eoh) {
 917                                 ret = fill_buffer(&io);
 918                                 if (ret <= 0) {
 919                                         break;
 920                                 }
 921                         }
 922                 } while (!eoh);
 923
 924                 if (!eoh) {
 925                         /* never got (complete) header */
 926                         fprintf(stderr, "incomplete response (ret = %zd) to %s\n", ret, url);
 927                         fprintf(stderr, "have:\n");
 928                         fwrite(io.response.buffer, io.response.len, 1, stderr);
 929                         exit(EXIT_FAILURE);
 930                 }
 931
 932                 header_len = (size_t)(eoh - io.response.buffer) + 4;
 933
 934                 parse_header(&io);
 935
 936                 switch (io.status_code) {
 937                         case 304:
 938                                 progressbar = 0;
 939                                 break;
 940                         case 301:
 941                         case 302:
 942                         case 303:
 943                         case 307:
 944                                 free(url);
 945                                 url = strdup(io.redirect);
 946                                 close(io.socket);
 947                                 continue;
 948                                 break;
 949                 }
 950
 951                 if (printstatus) {
 952                         printf("%d\n", io.status_code);
 953                         break;
 954                 }
 955                 if (showreq) {
 956                         fwrite(request.buffer, request.len, 1, stderr);
 957                 }
 958
 959                 if (head) {
 960                         io.response.len -= 2;
 961                 }
 962
 963                 if (outfile) {
 964                         out = open(outfile, O_WRONLY|O_CREAT|O_TRUNC, 0600);
 965                         if (out == -1) {
 966                                 perror("can't open output file:");
 967                                 exit(EXIT_FAILURE);
 968                         }
 969                 }
 970
 971                 if (progressbar) {
 972                         if (io.content_length) {
 973                                 fprintf(stderr, "(%lu) ", io.content_length);
 974                         }
 975                 }
 976
 977                 if (head) {
 978                         write(out, io.response.buffer, io.response.len);
 979                         break;
 980                 }
 981
 982                 if (raw) {
 983                         write(out, io.response.buffer, header_len);
 984                 }
 985                 tls_buffer_shift(&io.response, header_len);
 986
 987                 if (io.chunked) {
 988                         /* we've written out the head if needed, so
 989                          * what's in the response buffer is the
 990                          * chunked encoding, so just reassign that
 991                          * to the chunkbuf and reinit */
 992                         io.chunkbuf = io.response;
 993                         tls_buffer_init(&io.response, 0);
 994                         /* and put whatever we've got into the response
 995                          * buffer, may not be needed, fill buffer
 996                          * can handle it.
 997                          */
 998                         //unchunk(&io);
 999                 }
1000
1001                 do {
1002                         if (io.response.len) {
1003                                 if (io.content_length && io.response.len + io.received > io.content_length) {
1004                                         io.response.len = io.content_length - io.received;
1005                                         /* we just ignore trailing garbage */
1006                                 }
1007                                 write(out, io.response.buffer, io.response.len);
1008                                 io.received += io.response.len;
1009                                 ret = io.response.len;
1010                                 io.response.len = 0;
1011                         }
1012
1013                         if (progressbar) {
1014                                 if (io.content_length) {
1015                                         pdots(50, '.', total, total+ret,
1016                                                         io.content_length);
1017                                 } else {
1018                                         putc('\r', stderr);
1019                                         fprintf(stderr, "%zu", io.received);
1020                                 }
1021                                 total += ret;
1022                         }
1023                         if (head) {
1024                                 break;
1025                         }
1026                         if (io.content_length && io.received >= io.content_length) {
1027                                 break;
1028                         }
1029                         ret = fill_buffer(&io);
1030                 } while (ret > 0);
1031
1032                 //fprintf(stderr, "total received: %zu/%zu\n", io.received, io.content_length);
1033                 if (ret < 0) {
1034                         fprintf(stderr, "%s read error %zd\n", uri.scheme, ret);
1035                 }
1036                 if (io.last_modified != 0) {
1037                         struct timespec ts[2];
1038                         ts[0].tv_sec = 0; ts[0].tv_nsec = UTIME_OMIT;
1039                         ts[1].tv_sec = io.last_modified;
1040                         ts[1].tv_nsec = 0;
1041                         futimens(out, ts);
1042                 }
1043                 close(out);
1044                 tls_buffer_free(&io.response);
1045                 break;
1046         }
1047
1048         if (use_tls) {
1049                 tls_shutdown(clientssl);
1050                 tls_free(clientssl);
1051         }
1052
1053         close(sockfd);
1054         if (progressbar && io.status_code == 200) {
1055                 if (total == io.content_length || io.content_length == 0) {
1056                         fprintf(stderr, " done\n");
1057                 } else if (io.content_length != total) {
1058                         fprintf(stderr, "failed (%zu bytes read)\n", total);
1059                         io.status_code = 531; /* non official code */
1060                 }
1061         }
1062
1063         return io.status_code < 400 ? 0 : EXIT_FAILURE;
1064 }