pd.if.org Git - zpackage/blob - src/fetchurl.c

   1 #define _POSIX_C_SOURCE 200809L
   2
   3 #include <stdio.h>
   4 #include <sys/types.h>
   5 #include <sys/socket.h>
   6 #include <netinet/in.h>
   7 #include <netdb.h>
   8 #include <string.h>
   9 #include <strings.h>
  10 #include <stdlib.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <time.h>
  14 #include <sys/stat.h>
  15 #include <fcntl.h>
  16
  17 #include "tlse.h"
  18
  19 #define DEBUG(lvl, ...) if (debuglevel >= lvl ) { fprintf(stderr, __VA_ARGS__); }
  20
  21 struct tls_uri {
  22         char *scheme;
  23         char *userinfo;
  24         char *host;
  25         char *port;
  26         char *path;
  27         char *encoded_path;
  28         char *query;
  29         char *encoded_query;
  30         char *fragment;
  31 };
  32
  33 int tls_parse_uri(char *, struct tls_uri *);
  34 void tls_free_uri(struct tls_uri *);
  35
  36 int open_tcp_connection(char *host, int port);
  37
  38 /* if trustpolicy is 0, we just accept anything */
  39 int verify_trust(struct TLSContext *context, struct TLSCertificate **chain, int
  40                 len) {
  41         /* suppress unused */
  42         if (context || chain || len) {
  43                 return 0;
  44         }
  45
  46         return 0;
  47 }
  48
  49 static char hexchars[] = "0123456789abcdefABCDEF";
  50
  51 static void hex(char *dst, uint8_t *src, size_t len) {
  52         while (len--) {
  53                 dst[0] = hexchars[(src[0]>>4)&0xf];
  54                 dst[1] = hexchars[src[0]&0xf];
  55                 dst+=2;
  56                 src++;
  57         }
  58 }
  59
  60 static int debuglevel = 0;
  61
  62 #if 0
  63 static void hexbin(uint8_t *dst, unsigned char *src, size_t len) {
  64         size_t i;
  65         int x;
  66
  67         for (i=0; i<len; i+=2) {
  68                 sscanf((const char *)src+i, "%02x", &x);
  69                 dst[i/2] = x;
  70         }
  71 }
  72 #endif
  73
  74 char *my_getline(struct tls_buffer *b, int fd, size_t *size) {
  75         char *loc = 0;
  76         char buf[4096];
  77         ssize_t rv = 0;
  78
  79         while (b->error == 0) {
  80                 loc = memchr(b->buffer, '\n', b->len);
  81                 if (loc) {
  82                         *size = loc - b->buffer + 1;
  83                         return b->buffer;
  84                 } else {
  85                         rv = read(fd, buf, sizeof buf);
  86                         if (rv == -1) {
  87                                 return 0;
  88                         }
  89                         if (rv == 0) {
  90                                 break;
  91                         }
  92                         tls_buffer_append(b, buf, rv);
  93                 }
  94         }
  95         if (rv == 0) {
  96                 *size = b->len;
  97                 return b->buffer;
  98         }
  99         return 0;
 100 }
 101
 102 /*
 103  * We use a trust on first use policy.  The trust DB is a simple
 104  * file in /var/lib/zpm/known_hosts, or ~/.zpm/known_hosts, or ZPM_KNOWNHOSTS.
 105  * if -k is given, no verification is done
 106  */
 107 int verify_first(struct TLSContext *context, struct TLSCertificate **chain, int
 108                 certs) {
 109         int err;
 110         char *trustfile, *homedir = 0, *host, *fp;
 111         unsigned char certhash[65];
 112         int trustdb;
 113         struct tls_buffer tbuf;
 114
 115         char *line = 0;
 116         size_t len = 0;
 117
 118         if (certs == 0 || chain == 0) {
 119                 return 1;
 120         }
 121
 122         err = tls_certificate_is_valid(chain[0]);
 123         if (err) {
 124                 return err;
 125         }
 126
 127         if (context->sni) {
 128                 err = tls_certificate_valid_subject(chain[0], context->sni);
 129                 if (err) {
 130                         return err;
 131                 }
 132         }
 133
 134         hex(certhash, chain[0]->fp, 32);
 135         certhash[64] = 0;
 136
 137         trustfile = getenv("ZPM_KNOWNHOSTS");
 138         if (!trustfile) {
 139                 if (geteuid() == 0) {
 140                         trustfile = "/var/lib/zpm/known_hosts";
 141                 } else {
 142                         /* we could do this with a series of
 143                          * openat() calls instead of building
 144                          * up a string
 145                          */
 146                         trustfile = getenv("HOME");
 147                         if (!trustfile) {
 148                                 fprintf(stderr, "home = %s\n", trustfile);
 149                                 return 1;
 150                         }
 151                         len = snprintf(homedir, 0, "%s/.zpm/known_hosts", trustfile);
 152                         homedir = malloc(len+1);
 153                         if (!homedir) {
 154                                 return 1;
 155                         }
 156                         len = snprintf(homedir, len+1, "%s/.zpm/known_hosts", trustfile);
 157                         trustfile = homedir;
 158                 }
 159         }
 160         /* cert is valid on its face, so check against the trust db */
 161         trustdb = open(trustfile, O_RDWR|O_CREAT, 0600);
 162         if (trustdb == -1) {
 163                 fprintf(stderr, "cannot open trustdb %s: %s\n", trustfile, strerror(errno));
 164                 if (homedir) {
 165                         free(homedir);
 166                 }
 167                 return 1;
 168         }
 169
 170         if (homedir) {
 171                 free(homedir);
 172         }
 173
 174         len = 0;
 175         tls_buffer_init(&tbuf, 128);
 176         do {
 177                 char *off;
 178
 179                 tls_buffer_shift(&tbuf, len);
 180                 line = my_getline(&tbuf, trustdb, &len);
 181
 182                 if (!line || !len) {
 183                         break;
 184                 }
 185
 186                 fp = line;
 187                 while (isspace(*fp)) {
 188                         fp++;
 189                 }
 190                 if (*fp == '#') {
 191                         continue;
 192                 }
 193                 off = strchr(line, ':');
 194                 if (!off) {
 195                         continue;
 196                 }
 197                 host = off + 1;
 198                 *off = 0;
 199                 if (line[len-1] == '\n') {
 200                         line[len-1] = 0;
 201                 }
 202
 203                 if (strlen(fp) != 64) {
 204                         continue;
 205                 }
 206
 207                 if (len && line[len-1] == '\n') {
 208                         line[len-1] = 0;
 209                 }
 210
 211                 if (strcmp(context->sni, host) != 0) {
 212                         continue;
 213                 }
 214
 215                 int match = (memcmp(certhash, fp, 64) == 0);
 216                 if (!match) {
 217                         fprintf(stderr, "host %s certificate changed\n", host);
 218                         fprintf(stderr, "was %.64s\n", fp);
 219                         fprintf(stderr, "now %.64s\n", certhash);
 220                 }
 221
 222                 close(trustdb);
 223                 tls_buffer_free(&tbuf);
 224                 return match ? no_error : bad_certificate;
 225         } while (!tbuf.error);
 226
 227         /* got here, so we should be at EOF, so add this host to trust db */
 228         lseek(trustdb, 0, SEEK_END);
 229
 230         /* re-use the buffer so we only do one write */
 231         /* ignore errors, the cert is fine, we just can't update
 232          * the trustdb if there's errors here
 233          */
 234         tbuf.len = 0;
 235         tls_buffer_append(&tbuf, certhash, 64);
 236         tls_buffer_append_byte(&tbuf, ':');
 237         tls_buffer_append(&tbuf, context->sni, strlen(context->sni));
 238         tls_buffer_append_byte(&tbuf, '\n');
 239         write(trustdb, tbuf.buffer, tbuf.len);
 240         close(trustdb);
 241         tls_buffer_free(&tbuf);
 242
 243         return no_error;
 244 }
 245
 246 int verify_roots(struct TLSContext *context, struct TLSCertificate **chain, int
 247                 len) {
 248         int i, err;
 249
 250         if (chain) {
 251                 for (i = 0; i < len; i++) {
 252                         struct TLSCertificate *certificate = chain[i];
 253                         err = tls_certificate_is_valid(certificate);
 254                         if (err) {
 255                                 return err;
 256                         }
 257                 }
 258         }
 259
 260         err = tls_certificate_chain_is_valid(chain, len);
 261         if (err) {
 262                 return err;
 263         }
 264
 265         if (len > 0 && context->sni) {
 266                 err = tls_certificate_valid_subject(chain[0], context->sni);
 267                 if (err) {
 268                         return err;
 269                 }
 270         }
 271
 272         /* Perform certificate validation against ROOT CA */
 273         err = tls_certificate_chain_is_valid_root(context, chain, len);
 274         if (err) {
 275                 return err;
 276         }
 277
 278         return no_error;
 279 }
 280
 281 struct io {
 282         struct tls_buffer response;
 283         struct tls_buffer chunkbuf;
 284         struct TLSContext *tls;
 285         int socket;
 286         int chunked;
 287         int chunknum;
 288         size_t chunksize;
 289         size_t chunkleft;
 290         size_t chunktotal;
 291         size_t chunkbytesread;
 292         int status_code;
 293         time_t last_modified;
 294         time_t date;
 295         size_t content_length;
 296         size_t received;
 297         char *redirect;
 298 };
 299 ssize_t unchunk(struct io *io);
 300
 301 int month(char *m) {
 302         char *months[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul",
 303                 "Aug", "Sep", "Oct", "Nov", "Dec" };
 304         int i;
 305
 306         for (i=0; i < 12; i++) {
 307                 if (!strncasecmp(m, months[i], 3)) {
 308                         return i+1;
 309                 }
 310         }
 311         return 0;
 312 }
 313
 314 /* Wed, 06 Feb 2019 10:06:05 GMT */
 315 time_t parse_date(char *d) {
 316         struct tm tm = { 0 };
 317         int rv;
 318
 319         int h, m, s, dom, Y;
 320         char M[4];
 321         rv = sscanf(d, "%*[a-zA-Z,] %d %s %d %d:%d:%d", &dom, M, &Y, &h, &m, &s);
 322         if (rv == 6) {
 323                 tm.tm_year = Y - 1900;
 324                 tm.tm_hour = h;
 325                 tm.tm_min = m;
 326                 tm.tm_sec = s;
 327                 tm.tm_mon = month(M)-1;
 328                 tm.tm_mday = dom;
 329
 330                 return mktime(&tm);
 331         }
 332         return -1;
 333 }
 334
 335 char *find_header(struct io *io, char *header, size_t *len) {
 336         char *eoh, *soh;
 337         size_t hlen;
 338
 339         *len = 0;
 340
 341         hlen = strlen(header);
 342
 343         /* TODO can't do this, buffer may not be zero terminated */
 344         eoh = strstr(io->response.buffer, "\r\n\r\n");
 345         if (!eoh) {
 346                 return 0;
 347         }
 348
 349         soh = io->response.buffer;
 350         do {
 351                 /* skip the first line for some reason */
 352                 soh = strstr(soh, "\r\n");
 353                 if (soh == eoh) {
 354                         break;
 355                 }
 356                 soh += 2;
 357                 /* done if not enough room */
 358                 if (hlen > (size_t)(eoh - soh)) {
 359                         break;
 360                 }
 361                 if (!strncasecmp(soh, header, hlen)) {
 362                         break;
 363                 }
 364         } while (soh < eoh);
 365
 366         if (soh >= eoh) {
 367                 return 0;
 368         }
 369         eoh = strstr(soh, "\r\n");
 370         soh += hlen;
 371         while (soh < eoh && isspace(*soh)) {
 372                 soh++;
 373         }
 374         *len = (eoh - soh);
 375         return soh;
 376 }
 377
 378 void parse_header(struct io *io) {
 379         char *s = io->response.buffer;
 380         int code = 0;
 381         char *hval;
 382         size_t hlen;
 383
 384         while (!isspace(*s)) {
 385                 s++;
 386         }
 387         while (isspace(*s)) {
 388                 s++;
 389         }
 390         code = strtol(s, 0, 10);
 391         io->status_code = code;
 392
 393         hval = find_header(io, "Date:", &hlen);
 394         if (hval) {
 395                 hval[hlen] = 0;
 396                 io->date = parse_date(hval);
 397                 hval[hlen] = '\r';
 398         }
 399         hval = find_header(io, "Last-Modified:", &hlen);
 400         if (hval) {
 401                 hval[hlen] = 0;
 402                 io->last_modified = parse_date(hval);
 403                 hval[hlen] = '\r';
 404         }
 405
 406         hval = find_header(io, "Content-Length:", &hlen);
 407         if (hval) {
 408                 hval[hlen] = 0;
 409                 io->content_length = strtoul(hval, 0, 10);
 410                 hval[hlen] = '\r';
 411         }
 412
 413         hval = find_header(io, "Transfer-Encoding:", &hlen);
 414         if (hval) {
 415                 hval[hlen] = 0;
 416                 io->content_length = strtoul(hval, 0, 10);
 417                 if (!strcmp(hval, "chunked")) {
 418                         io->chunked = 1;
 419                 }
 420                 hval[hlen] = '\r';
 421         }
 422
 423         switch (code) {
 424                 case 301:
 425                 case 302:
 426                 case 303:
 427                 case 307:
 428                         DEBUG(1, "looking for Location header\n");
 429                         hval = find_header(io, "Location:", &hlen);
 430                         if (hval) {
 431                                 io->redirect = strndup(hval, hlen);
 432                         }
 433                         break;
 434                 default:
 435                         break;
 436         }
 437
 438 }
 439
 440 /* fill buffer needs to put bytes into the response buffer
 441  * if the transfer encoding is chunked, it will need to
 442  * put the bytes into the chunkbuf first, then call
 443  * unchunk.  if unchunk return 0, then it needs more data,
 444  * otherwise unchunk returns the number of bytes transferred
 445  */
 446
 447 ssize_t fill_buffer(struct io *io) {
 448         unsigned char buffer[4096];
 449         ssize_t ret = 0;
 450
 451         ret = unchunk(io);
 452
 453         while (ret == 0) {
 454                 if (io->tls) {
 455                         ret = tls_read(io->tls, buffer, sizeof buffer);
 456                 } else {
 457                         ret = read(io->socket, buffer, sizeof buffer);
 458                 }
 459
 460                 if (ret <= 0) {
 461                         break;
 462                 }
 463
 464                 if (io->chunked) {
 465                         tls_buffer_append(&io->chunkbuf, buffer, ret);
 466                         //fwrite(buffer, ret, 1, stderr);
 467                         ret = unchunk(io);
 468                         if (ret != 0 || io->chunksize == 0) {
 469                                 break;
 470                         }
 471                 } else {
 472                         tls_buffer_append(&io->response, buffer, ret);
 473                         break;
 474                 }
 475         }
 476
 477         return ret;
 478 }
 479
 480 /* essentially memmem */
 481 void *lookfor(const void *buf, size_t buflen, const void *pattern, size_t len) {
 482       const char *bf = buf;
 483       const char *pt = pattern;
 484       const char *p = bf;
 485
 486       while (len <= (buflen - (p - bf))) {
 487             if ((p = memchr(p, *pt, buflen - (p - bf))) != 0) {
 488                   if (memcmp(p, pattern, len) == 0) {
 489                         return (void *)p;
 490                   } else {
 491                           p++;
 492                   }
 493             } else {
 494                     break;
 495             }
 496       }
 497       return NULL;
 498 }
 499
 500 /* returns read chunksize, unshifts the line */
 501 ssize_t read_chunksize(struct io *io) {
 502         char *cr;
 503         ssize_t cs;
 504
 505         //fwrite(io->chunkbuf.buffer, io->chunkbuf.len, 1, stderr);
 506
 507         /* there could be up to two leading bytes */
 508         if (io->chunkbuf.len >= 2 && io->chunkbuf.buffer[0] == '\r' && io->chunkbuf.buffer[1] == '\n') {
 509                 tls_buffer_shift(&io->chunkbuf, 2);
 510         }
 511
 512         cr = lookfor(io->chunkbuf.buffer, io->chunkbuf.len, "\r\n", 2);
 513
 514         if (cr == 0) {
 515                 return -1;
 516         }
 517
 518         cs = strtol(io->chunkbuf.buffer, 0, 16);
 519         tls_buffer_shift(&io->chunkbuf, cr - io->chunkbuf.buffer + 2);
 520
 521         return cs;
 522 }
 523
 524 /* unchunk's job is to move bytes from the chunk buf to the response buf */
 525 /* return bytes from chunk, 0 if unable.  once last chunk, changed chunked
 526  * to 0?
 527  */
 528 ssize_t unchunk(struct io *io) {
 529         ssize_t bytes_to_move = 0;
 530         ssize_t chunksize;
 531
 532         if (!io || !io->chunked) {
 533                 return 0;
 534         }
 535
 536         if (io->chunkleft == 0) {
 537                 chunksize = read_chunksize(io);
 538                 if (chunksize == -1) {
 539                         return 0;
 540                 }
 541                 io->chunksize = chunksize;
 542                 if (io->chunksize == 0) {
 543                         /* end of chunked data */
 544                         io->chunked = 0;
 545                         return 0;
 546                 }
 547                 io->chunknum++;
 548                 io->chunkleft = io->chunksize;
 549                 io->chunktotal += io->chunksize;
 550         }
 551
 552         if (io->chunkbuf.len == 0) {
 553                 /* need more bytes */
 554                 return 0;
 555         }
 556
 557         bytes_to_move = io->chunkbuf.len < io->chunkleft ? io->chunkbuf.len : io->chunkleft;
 558
 559         tls_buffer_append(&io->response, io->chunkbuf.buffer, bytes_to_move);
 560         io->chunkleft -= bytes_to_move;
 561         io->chunkbytesread += bytes_to_move;
 562
 563         /* chunk is terminated with a crlf */
 564         //tls_buffer_shift(&io->chunkbuf, bytes_to_move + io->chunkleft ? 0 : 2);
 565         tls_buffer_shift(&io->chunkbuf, bytes_to_move);
 566
 567         return bytes_to_move;
 568 }
 569
 570 #if 0
 571 char *nextline(struct io *io) {
 572         char *eol = 0;;
 573
 574         eol = memchr(io->response.buffer, '\n', io.response.size);
 575         while (eol == 0) {
 576                 fill_buffer(io);
 577                 eol = memchr(io->response.buffer, '\n', io.response.size);
 578         }
 579         if (eol) {
 580
 581 }
 582 #endif
 583
 584 void append_header(struct tls_buffer *buf, char *header, char *val) {
 585         tls_buffer_append(buf, header, strlen(header));
 586         tls_buffer_append(buf, ": ", 2);
 587         tls_buffer_append(buf, val, strlen(val));
 588         tls_buffer_append(buf, "\r\n", 2);
 589 }
 590
 591 void append_timeheader(struct tls_buffer *buf, char *header, time_t ts) {
 592         char timestr[80];
 593         struct tm *tm;
 594
 595         tm = gmtime(&ts);
 596
 597         strftime(timestr, sizeof timestr, "%a, %d %b %Y %H:%M:%S GMT", tm);
 598         append_header(buf, header, timestr);
 599 }
 600
 601 static void pdots(int len, int ch, unsigned long was, unsigned long now,
 602                 unsigned long total) {
 603         was = len * was / total;
 604         if (now > total) {
 605                 now = total;
 606         }
 607         now = len * now / total;
 608         while (was++ < now) {
 609                 putc(ch,stderr);
 610         }
 611 }
 612
 613 static void fake_header(struct io *io, int fd) {
 614         struct stat st;
 615         int code = 200, rv;
 616         char *message, codestr[5], length[32];
 617         struct tls_buffer *hdr = &io->response;
 618
 619         if (fd == -1) {
 620                 switch (errno) {
 621                         case EACCES: code = 403; break;
 622                         case ENOENT: code = 404; break;
 623                         default: code = 500; break;
 624                 }
 625         } else {
 626                 rv = fstat(fd, &st);
 627                 if (rv == -1) {
 628                         code = 500;
 629                 }
 630         }
 631
 632         if (io->last_modified >= st.st_mtime) {
 633                 code = 304;
 634         }
 635
 636         switch (code) {
 637                 case 200: message = "OK"; break;
 638                 case 304: message = "Not Modified"; break;
 639                 case 403: message = "Forbidden"; break;
 640                 case 404: message = "Not Found"; break;
 641                 case 500: message = "Internal Server Error"; break;
 642                 default: break;
 643         }
 644         sprintf(codestr, "%0.3d ", code);
 645         tls_buffer_append(hdr, "HTTP/1.1 ", 9);
 646         tls_buffer_append(hdr, codestr, 4);
 647         tls_buffer_append_str(hdr, message);
 648         tls_buffer_append(hdr, "\r\n", 2);
 649
 650         append_timeheader(hdr, "Date", time(NULL));
 651         append_header(hdr, "Server", "zpm-fetchurl/0.9");
 652         if (code < 400) {
 653                 append_timeheader(hdr, "Last-Modified", st.st_mtime);
 654                 sprintf(length, "%zu", st.st_size);
 655                 append_header(hdr, "Content-Length", length);
 656         }
 657
 658         append_header(hdr, "Connection", "close");
 659         append_header(hdr, "Content-Type", "application/octet-stream");
 660         tls_buffer_append(hdr, "\r\n", 2);
 661 }
 662
 663 char *pathlast(char *path) {
 664         char *last = 0;
 665         size_t len = 0;
 666
 667         if (path == 0) {
 668                 return 0;
 669         }
 670
 671         while (*path == '/') {
 672                 path++;
 673         }
 674
 675         do {
 676                 last = path;
 677                 len = 0;
 678                 while (*path && *path != '/') {
 679                         path++;
 680                         len++;
 681                 }
 682                 while (*path == '/') {
 683                         path++;
 684                 }
 685         } while (*path);
 686
 687         if (len == 0) {
 688                 return 0;
 689         }
 690
 691         return strndup(last, len);
 692 }
 693
 694 static time_t file_mtime(char *path) {
 695         struct stat st;
 696         int rv;
 697
 698         rv = stat(path, &st);
 699         if (rv == -1) {
 700                 if (errno == ENOENT) {
 701                         return 0;
 702                 }
 703                 perror("stat failed:");
 704                 return -1;
 705         }
 706
 707         return st.st_mtime;
 708 }
 709
 710 int main(int ac, char *av[]) {
 711         int sockfd, port = -1, rv;
 712         ssize_t ret;
 713         int option;
 714 #if 0
 715         char msg[] = "GET %s HTTP/1.1\r\nHost: %s:%i\r\nConnection: close\r\n\r\n";
 716         char msg2[] = "GET %s HTTP/1.1\r\nHost: %s:%i\r\nLast-Modified: %s\r\nConnection: close\r\n\r\n";
 717         char msg_buffer[1024];
 718 #endif
 719         char *req_file = 0;
 720         char *host = 0;
 721         struct tls_uri uri = { 0 };
 722         char *outfile = 0;
 723         int raw = 0, head = 0;
 724         int out = 1; /* output file descriptor */
 725         int use_tls = 0;
 726         struct io io = { {0}, {0}, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
 727         struct TLSContext *clientssl = 0;
 728         int failsilent = 0;
 729         char *lmfile = 0;
 730         int progressbar = 0;
 731         struct tls_buffer request;
 732         char lmtime[80];
 733         char *eoh = 0;
 734         char *user_agent = 0;
 735         size_t total = 0;
 736         size_t header_len;
 737         char *url = 0;
 738         int redirs = 0, redirlimit = 50, printstatus = 0, showreq = 0;
 739         int verifypolicy = 1, calcoutfile = 0, ifnewer = 0;
 740
 741         ltc_mp = tfm_desc;
 742
 743         while ((option = getopt(ac, av, "do:OrIfz:np#RL:SkKU:")) != -1) {
 744                 switch (option) {
 745                         case 'd': debuglevel++; break;
 746                         case 'o': outfile = optarg; break;
 747                         case 'O': calcoutfile = 1; break;
 748                         case 'S': printstatus = 1; head = 1; break;
 749                         case 'k': verifypolicy = 0; break;
 750                         case 'K': verifypolicy = 2; break;
 751                         case 'U': user_agent = optarg; break;
 752                         case 'I': head = 1;
 753                         case 'r': raw = 1; break;
 754                         case 'R': showreq = 1; break;
 755                         case 'f': failsilent = 1; break;
 756                         case 'z': lmfile = optarg; break;
 757                         case 'n': ifnewer = 1; break;
 758                         case 'L': redirlimit = strtol(optarg, 0, 10); break;
 759                         case 'p':
 760                         case '#': progressbar = 1; break;
 761                         default:
 762                                   exit(EXIT_FAILURE);
 763                                   break;
 764                 }
 765         }
 766
 767         if (ac < optind) {
 768                 fprintf(stderr, "Usage: %s uri\n", av[0]);
 769                 exit(EXIT_FAILURE);
 770         }
 771
 772         url = strdup(av[optind]);
 773         if (!url) {
 774                 exit(EXIT_FAILURE);
 775         }
 776
 777         io.last_modified = 0;
 778
 779         if (calcoutfile && !outfile) {
 780                 tls_parse_uri(url, &uri);
 781                 outfile = pathlast(uri.path);
 782                 /* outfile leaks memory here, so if this
 783                  * were turned into a library function,
 784                  * we'd need to track it
 785                  */
 786                 if (!outfile) {
 787                         fprintf(stderr, "unable to determine outfile\n");
 788                         exit(EXIT_FAILURE);
 789                 }
 790         }
 791
 792         if (ifnewer && outfile && !lmfile) {
 793                 lmfile = outfile;
 794         }
 795
 796         if (lmfile) {
 797                 struct tm *mtime;
 798                 time_t ts;
 799
 800                 ts = file_mtime(lmfile);
 801
 802                 if (ts == -1) {
 803                         exit(EXIT_FAILURE);
 804                 } else if (ts != 0) {
 805                         io.last_modified = ts;
 806                         mtime = gmtime(&ts);
 807                         strftime(lmtime, sizeof lmtime, "%a, %d %b %Y %H:%M:%S GMT", mtime);
 808                 } else {
 809                         lmfile = 0;
 810                 }
 811         }
 812
 813         signal(SIGPIPE, SIG_IGN);
 814
 815         tls_buffer_init(&io.response, 0);
 816         tls_buffer_init(&request, 128);
 817
 818         while (redirs++ <= redirlimit) {
 819                 tls_free_uri(&uri);
 820                 io.response.len = 0;
 821                 io.chunked = 0;
 822                 request.len = 0;
 823                 eoh = 0;
 824
 825                 tls_parse_uri(url, &uri);
 826                 host = uri.host;
 827                 port = atoi(uri.port);
 828                 req_file = uri.path;
 829
 830                 /* construct request */
 831                 if (head) {
 832                         tls_buffer_append(&request, "HEAD ", 5);
 833                 } else {
 834                         tls_buffer_append(&request, "GET ", 4);
 835                 }
 836                 tls_buffer_append(&request, uri.encoded_path, strlen(uri.encoded_path));
 837                 if (uri.encoded_query) {
 838                         tls_buffer_append(&request, "?", 1);
 839                         tls_buffer_append(&request, uri.encoded_query, strlen(uri.encoded_query));
 840                 }
 841                 tls_buffer_append(&request, " HTTP/1.1\r\n", 11);
 842
 843                 append_header(&request, "Host", host);
 844                 if (user_agent) {
 845                         append_header(&request, "User-Agent", user_agent);
 846                 }
 847                 append_header(&request, "Accept", "*/*");
 848                 //append_header(&request, "Accept-Encoding", "chunked, identity;q=0.5");
 849                 append_header(&request, "Connection", "close");
 850                 if (lmfile) {
 851                         append_header(&request, "If-Modified-Since", lmtime);
 852                 }
 853                 tls_buffer_append(&request, "\r\n", 2);
 854
 855                 if (!strcmp(uri.scheme, "https")) {
 856                         use_tls = 1;
 857
 858                         DEBUG(1, "creating tls context\n");
 859                         clientssl = tls_create_context(TLS_CLIENT, TLS_V12);
 860
 861                         /* optionally, we can set a certificate validation
 862                          * callback function if set_verify is not called, and
 863                          * root ca is set, `tls_default_verify` will be used
 864                          * (does exactly what `verify` does in this example)
 865                          */
 866                         if (verifypolicy == 2) {
 867                                 char *cert_path = 0;
 868                                 cert_path = getenv("ZPM_CERTFILE");
 869                                 if (!cert_path) {
 870                                         cert_path = "/var/lib/zpm/roots.pem";
 871                                 }
 872                                 rv = tls_load_root_file(clientssl, cert_path);
 873                                 if (rv == -1) {
 874                                         fprintf(stderr, "Error loading root certs\n");
 875                                         return 1;
 876                                 }
 877                                 DEBUG(1, "verifying ssl cert via roots\n");
 878                                 tls_set_verify(clientssl, verify_roots);
 879                         } else if (verifypolicy == 1) {
 880                                 DEBUG(1, "verifying ssl cert via first use\n");
 881                                 tls_set_verify(clientssl, verify_first);
 882                                 DEBUG(1, "verified ssl cert via first use\n");
 883                         } else {
 884                                 DEBUG(1, "verifying ssl cert via trust\n");
 885                                 tls_set_verify(clientssl, verify_trust);
 886                         }
 887
 888                         if (!clientssl) {
 889                                 fprintf(stderr, "Error initializing client context\n");
 890                                 return -1;
 891                         }
 892                         tls_sni_set(clientssl, uri.host);
 893                         DEBUG(1, "set sni to %s\n", uri.host);
 894                         clientssl->sync = 1;
 895                         io.tls = clientssl;
 896                         sockfd = open_tcp_connection(host, port);
 897                         DEBUG(1, "opened tcp socket fd %d\n", sockfd);
 898                         if (sockfd < 0) {
 899                                 perror("can't open connection");
 900                                 exit(EXIT_FAILURE);
 901                         }
 902                         tls_set_fd(clientssl, sockfd);
 903                         if ((rv = tls_connect(clientssl)) != 1) {
 904                                 fprintf(stderr, "Handshake Error %i\n", rv);
 905                                 return 1;
 906                         }
 907                         ret = tls_write(clientssl, request.buffer, request.len);
 908                 } else if (!strcmp(uri.scheme, "http")) {
 909                         sockfd = open_tcp_connection(host, port);
 910                         if (sockfd < 0) {
 911                                 perror("can't open connection");
 912                                 exit(EXIT_FAILURE);
 913                         }
 914                         ret = write(sockfd, request.buffer, request.len);
 915                 } else if (!strcmp(uri.scheme, "file")) {
 916                         sockfd = open(uri.path, O_RDONLY);
 917                         fake_header(&io, sockfd);
 918                         ret = 0;
 919                 } else {
 920                         fprintf(stderr, "scheme %s unknown\n", uri.scheme);
 921                         exit(EXIT_FAILURE);
 922                 }
 923
 924                 DEBUG(1, "wrote http request\n");
 925                 if (ret == -1) {
 926                         fprintf(stderr, "unable to write http request: %s\n", strerror(errno));
 927                         exit(EXIT_FAILURE);
 928                 }
 929
 930                 io.socket = sockfd;
 931
 932                 eoh = 0;
 933                 do {
 934                         if (io.response.len >= 4) {
 935                                 eoh = strstr(io.response.buffer, "\r\n\r\n");
 936                         }
 937                         if (!eoh) {
 938                                 DEBUG(1, "filling buffer\n");
 939                                 ret = fill_buffer(&io);
 940                                 if (ret <= 0) {
 941                                         break;
 942                                 }
 943                         }
 944                 } while (!eoh);
 945                 DEBUG(1, "got response\n");
 946
 947                 if (!eoh) {
 948                         /* never got (complete) header */
 949                         fprintf(stderr, "incomplete response (ret = %zd) to %s\n", ret, url);
 950                         fprintf(stderr, "have:\n");
 951                         fwrite(io.response.buffer, io.response.len, 1, stderr);
 952                         exit(EXIT_FAILURE);
 953                 }
 954
 955                 header_len = (size_t)(eoh - io.response.buffer) + 4;
 956
 957                 parse_header(&io);
 958                 DEBUG(1, "parsed response header, code %d\n", io.status_code);
 959
 960                 switch (io.status_code) {
 961                         case 304:
 962                                 progressbar = 0;
 963                                 break;
 964                         case 301:
 965                         case 302:
 966                         case 303:
 967                         case 307:
 968                                 DEBUG(1, "redirecting to %s\n", io.redirect);
 969                                 free(url);
 970                                 url = strdup(io.redirect);
 971                                 DEBUG(1, "redirecting to %s\n", url);
 972                                 close(io.socket);
 973                                 continue;
 974                                 break;
 975                 }
 976
 977                 if (printstatus) {
 978                         printf("%d\n", io.status_code);
 979                         break;
 980                 }
 981                 if (showreq) {
 982                         fwrite(request.buffer, request.len, 1, stderr);
 983                 }
 984
 985                 if (head) {
 986                         io.response.len -= 2;
 987                         write(out, io.response.buffer, io.response.len);
 988                         break;
 989                 }
 990
 991                 if (io.status_code == 304) {
 992                         break;
 993                 }
 994
 995                 if (outfile) {
 996                         out = open(outfile, O_WRONLY|O_CREAT|O_TRUNC, 0600);
 997                         if (out == -1) {
 998                                 perror("can't open output file:");
 999                                 exit(EXIT_FAILURE);
1000                         }
1001                 }
1002
1003                 if (progressbar) {
1004                         if (io.content_length) {
1005                                 fprintf(stderr, "(%lu) ", io.content_length);
1006                         }
1007                 }
1008
1009                 if (raw) {
1010                         write(out, io.response.buffer, header_len);
1011                 }
1012                 tls_buffer_shift(&io.response, header_len);
1013
1014                 if (io.chunked) {
1015                         /* we've written out the head if needed, so
1016                          * what's in the response buffer is the
1017                          * chunked encoding, so just reassign that
1018                          * to the chunkbuf and reinit */
1019                         io.chunkbuf = io.response;
1020                         tls_buffer_init(&io.response, 0);
1021                         /* and put whatever we've got into the response
1022                          * buffer, may not be needed, fill buffer
1023                          * can handle it.
1024                          */
1025                         //unchunk(&io);
1026                 }
1027
1028                 do {
1029                         size_t before = io.received;
1030                         if (io.response.len) {
1031                                 if (io.content_length && io.response.len + io.received > io.content_length) {
1032                                         io.response.len = io.content_length - io.received;
1033                                         /* we just ignore trailing garbage */
1034                                 }
1035                                 write(out, io.response.buffer, io.response.len);
1036                                 io.received += io.response.len;
1037                                 ret = io.response.len;
1038                                 io.response.len = 0;
1039                         }
1040
1041                         if (progressbar) {
1042                                 if (io.content_length) {
1043                                         pdots(50, '.', before, io.received,
1044                                                         io.content_length);
1045                                 } else {
1046                                         putc('\r', stderr);
1047                                         fprintf(stderr, "%zu", io.received);
1048                                 }
1049                                 total += ret;
1050                         }
1051                         if (head) {
1052                                 break;
1053                         }
1054                         if (io.content_length && io.received >= io.content_length) {
1055                                 break;
1056                         }
1057                         ret = fill_buffer(&io);
1058                 } while (ret > 0);
1059
1060                 //fprintf(stderr, "total received: %zu/%zu\n", io.received, io.content_length);
1061                 if (ret < 0) {
1062                         fprintf(stderr, "%s read error %zd\n", uri.scheme, ret);
1063                 }
1064                 if (io.last_modified != 0) {
1065                         struct timespec ts[2];
1066                         ts[0].tv_sec = 0; ts[0].tv_nsec = UTIME_OMIT;
1067                         ts[1].tv_sec = io.last_modified;
1068                         ts[1].tv_nsec = 0;
1069                         futimens(out, ts);
1070                 }
1071                 close(out);
1072                 tls_buffer_free(&io.response);
1073                 break;
1074         }
1075
1076         if (use_tls) {
1077                 tls_shutdown(clientssl);
1078                 tls_free(clientssl);
1079         }
1080
1081         close(sockfd);
1082         if (progressbar && io.status_code == 200) {
1083                 if (io.received == io.content_length || io.content_length == 0) {
1084                         fprintf(stderr, " done\n");
1085                 } else if (io.content_length != io.received) {
1086                         fprintf(stderr, "failed (%zu bytes read)\n", total);
1087                         io.status_code = 531; /* non official code */
1088                 }
1089         }
1090
1091         return io.status_code < 400 ? 0 : EXIT_FAILURE;
1092 }