pd.if.org Git - zpackage/blob - t/ctap/prove.c

   1 #define _POSIX_C_SOURCE 200809L
   2
   3 #include <stdlib.h>
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <strings.h>
   7 #include <unistd.h>
   8 #include <errno.h>
   9 #include <sys/types.h>
  10 #include <sys/wait.h>
  11 #include <regex.h>
  12
  13 #define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) )
  14 #define WARN(x) fprintf(stderr, "%s:%d %s\n", __FILE__, __LINE__, x)
  15
  16 struct result {
  17         int test; /* i.e. the number of the test */
  18         char *name; /* name of the test, malloc */
  19         int pass; /* if the test passed */
  20         struct result *next; /* so we can make a linked list */
  21 };
  22
  23 struct testrun {
  24         char *name;
  25         int readstdin;
  26         int version;
  27         int plan;
  28         int ran;
  29         int pass;
  30         int fail;
  31         int skip;
  32         int todo, todo_pass, todo_fail;
  33         int bailout;
  34         struct result *results;
  35         struct result *last; /* last result, so we append rather than push */
  36         int interactive;
  37 };
  38
  39 int lookfor(char *needle, char *haystack, size_t n) {
  40         return strncasecmp(needle, haystack, n) == 0;
  41 }
  42
  43 int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) {
  44         return regexec(preg, string, 4, pmatch, 0) == 0;
  45 }
  46
  47 /* write n backspaces to stdout */
  48 void backup(int n) {
  49         while (n--) putchar('\b');
  50 }
  51
  52 void spaces(int n) {
  53         while (n--) putchar(' ');
  54 }
  55
  56 int runone(struct testrun *run) {
  57         int pipefd[2];
  58         pid_t cpid;
  59         FILE *tap;
  60         char *line = 0;
  61         ssize_t nread;
  62         size_t len = 0;
  63         int written = 0;
  64
  65         if (!run->readstdin) {
  66                 if (pipe(pipefd) == -1) {
  67                         perror("pipe");
  68                         exit(EXIT_FAILURE);
  69                 }
  70
  71                 cpid = fork();
  72                 if (cpid == -1) {
  73                         perror("fork");
  74                         exit(EXIT_FAILURE);
  75                 }
  76
  77                 if (cpid == 0) {
  78                         /* child */
  79                         close(pipefd[0]); /* close read end */
  80                         close(1); /* close stdout */
  81                         dup(pipefd[1]); /* set stdout to the write end of the pipe */
  82                         execlp(run->name, run->name, NULL);
  83                         /* only gets here if exec failed */
  84                         printf("Bail Out! exec %s failed: %s\n", run->name, strerror(errno));
  85                         exit(EXIT_FAILURE);
  86                 }
  87
  88                 close(pipefd[1]);
  89                 /* parent continue on */
  90                 tap = fdopen(pipefd[0], "r");
  91                 if (tap == NULL) {
  92                         perror("tap is null:");
  93                         exit(EXIT_FAILURE);
  94                 }
  95         } else {
  96                 tap = stdin;
  97         }
  98
  99         regex_t plan, diagnostic, bail, test, directive, version;
 100         regmatch_t match[4];
 101
 102         regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED);
 103         regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED);
 104         regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED|REG_ICASE);
 105         regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED);
 106
 107         regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED);
 108         regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?",
 109                         REG_EXTENDED|REG_ICASE);
 110
 111
 112                 /* description: ok \d+([^#]+) */
 113                 /* directive: #.+ */
 114                 /* todo: # todo */
 115                 /* skip: # skip */
 116
 117                 /* bail out: ^Bail out! */
 118                 /* diagnostic: ^#.+ */
 119                 /* start of yaml: ^\s+--- */
 120                 /* end of yaml: ^\s+... */
 121                 /* between yaml lines must be indented */
 122                 /* anything else is an error */
 123
 124         while ((nread = getline(&line, &len, tap)) != -1) {
 125                 /* version: ^TAP version \d+ */
 126
 127                 /* plan: ^\d+\.\.\d+ */
 128                 if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) {
 129                         run->plan = atoi(line+match[1].rm_so);
 130                         /* TODO plans can have a diag skip reason */
 131                 } else
 132
 133                 /* ok: ^ok \d+ */
 134                 /* not ok: ^not ok \d+ */
 135                 if (rmatch(&test, line, match)) {
 136                         int test;
 137                         int todo = 0;
 138                         int skip = 0;
 139                         int pass = 0;
 140
 141                         run->ran++;
 142
 143                         if (run->interactive) {
 144                                 backup(written);
 145                                 written = printf("%d/%d", run->ran, run->plan);
 146                                 fflush(stdout);
 147                         }
 148
 149
 150                         /* if the "not" match fails, the test passed */
 151                         pass = match[1].rm_so == -1;
 152
 153                         /* test number */
 154                         test = atoi(line+match[3].rm_so);
 155
 156                         /* directive */
 157                         if (rmatch(&directive, line, match)) {
 158                                 todo = lookfor("todo", line+match[1].rm_so, 4);
 159                                 skip = lookfor("skip", line+match[1].rm_so, 4);
 160                         }
 161
 162                         /* check for diagnostics */
 163                         if (test != run->ran) {
 164                                 fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test);
 165                         }
 166
 167                         /* pass, todo, or skip are all pass */
 168                         if (pass || todo || skip) {
 169                                 run->pass++;
 170                         } else {
 171                                 run->fail++;
 172                         }
 173
 174                         if (todo && pass) {
 175                                 run->todo_pass++;
 176                         }
 177
 178                         if (skip) {
 179                                 run->skip++;
 180                         }
 181
 182                         if (!pass && !todo) {
 183                                 struct result *r;
 184                                 r = malloc(sizeof *r);
 185
 186                                 r->test = run->ran;
 187                                 r->pass = 0;
 188                                 if (run->last) {
 189                                         run->last->next = r;
 190                                 }
 191                                 if (!run->results) {
 192                                         run->results = r;
 193                                 }
 194                                 run->last = r;
 195                                 /* push test number onto fail list */
 196                         }
 197
 198                 }
 199                 else if (rmatch(&diagnostic, line, match)) {
 200
 201                 }
 202                 else if (rmatch(&version, line, match)) {
 203                         run->version = atoi(line + match[1].rm_so);
 204                 }
 205                 else if (rmatch(&bail, line, match)) {
 206                         run->bailout = 1;
 207                 }
 208                 else {
 209                         printf("Unknown line: %s", line);
 210                 }
 211
 212                 /* not ok: ^not ok \d+ */
 213                 /* description: ok \d+([^#]+) */
 214                 /* directive: #.+ */
 215                 /* todo: # todo */
 216                 /* skip: # skip */
 217
 218                 /* bail out: ^Bail out! */
 219                 /* diagnostic: ^#.+ */
 220                 /* start of yaml: ^\s+--- */
 221                 /* end of yaml: ^\s+... */
 222                 /* between yaml lines must be indented */
 223                 /* anything else is an error */
 224
 225         }
 226
 227         regfree(&plan);
 228         regfree(&test);
 229
 230         free(line);
 231         if (!run->readstdin) {
 232                 fclose(tap);
 233         }
 234         wait(NULL);
 235
 236         if (run->ran < run->plan) {
 237                 run->fail += run->plan - run->ran;
 238         }
 239
 240         if (run->interactive) {
 241                 backup(written);
 242                 spaces(written);
 243                 backup(written);
 244                 fflush(stdout);
 245         }
 246
 247         return 0;
 248 }
 249
 250 void json(struct testrun *total, struct testrun *runs, int nruns) {
 251         if (total && runs && nruns) {
 252                 return;
 253         }
 254         return;
 255 }
 256
 257 int main(int ac, char *av[]) {
 258         int maxnamelen = 0;
 259         ssize_t len;
 260         int i, j;
 261         struct testrun total = { 0 };
 262         struct testrun run = { 0 };
 263         struct testrun clear = { 0 };
 264         struct testrun *runs;
 265         int opt;
 266         char *stdinname = "<stdin>";
 267         int stdinnamelen = 0;
 268
 269         /*
 270          * -j json output
 271          * -p progress output
 272          * -i interactive, regardless of tty
 273          * -s summary text
 274          * -d detail text
 275          */
 276
 277         int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0;
 278
 279         interactive = isatty(1);
 280
 281         /* TODO -jj turn off everything else? */
 282         /* TODO -cC color escapes */
 283         while ((opt = getopt(ac, av, "jJpPiIsSdDn:")) != -1) {
 284                 switch (opt) {
 285                         case 'j': json = 1; break;
 286                         case 'J': json = 0; break;
 287                         case 'p': progress = 1; break;
 288                         case 'P': progress = 0; break;
 289                         case 'i': interactive = 1; break;
 290                         case 'I': interactive = 0; break;
 291                         case 's': summary = 1; break;
 292                         case 'S': summary = 0; break;
 293                         case 'd': detail = 1; break;
 294                         case 'D': detail = 0; break;
 295                         case 'n': stdinname = optarg;
 296                                   break;
 297                         default: /* '?' */
 298                                 fprintf(stderr, "Usage: %s [-jpisd] file...\n",
 299                                                 av[0]);
 300                                 exit(EXIT_FAILURE);
 301                 }
 302         }
 303         stdinnamelen = strlen(stdinname);
 304
 305         runs = malloc(ac * sizeof *runs);
 306
 307         for (i=optind; i<ac; i++) {
 308                 if (!strcmp(av[i], "-")) {
 309                         maxnamelen = GREATER(stdinnamelen, maxnamelen);
 310                 } else {
 311                         len = strlen(av[i]);
 312                         maxnamelen = GREATER(len, maxnamelen);
 313                 }
 314         }
 315
 316         for (i=optind; i<ac; i++) {
 317                 run = clear;
 318                 run.interactive = interactive;
 319
 320                 if (!strcmp(av[i], "-")) {
 321                         run.readstdin = 1;
 322                         run.name = stdinname;
 323                 } else {
 324                         run.readstdin = 0;
 325                         run.name = av[i];
 326                 }
 327
 328                 if (progress) {
 329                         printf("%s ", run.name);
 330                         len = strlen(run.name);
 331                         for (j=len;j<maxnamelen+2;j++) {
 332                                 putchar('.');
 333                         }
 334                         putchar(' ');
 335                         fflush(stdout);
 336                 }
 337
 338                 runone(&run);
 339                 total.plan += run.plan;
 340                 total.ran += run.ran;
 341                 total.pass += run.pass;
 342                 total.fail += run.fail;
 343                 total.skip += run.skip;
 344
 345                 if (progress) {
 346                         if (run.bailout) {
 347                                 printf("Bail out!\n");
 348                         } else {
 349                                 if (run.fail > 0 ) printf("not ");
 350                                 printf("ok\n");
 351                         }
 352                 }
 353
 354                 runs[i-optind] = run;
 355         }
 356
 357         if (summary) {
 358                 printf("ran: %d/%d, pass: %d, fail: %d, skip: %d",
 359                                 total.ran, total.plan, total.pass, total.fail,
 360                                 total.skip
 361                       );
 362                 if (total.plan > 0) {
 363                         printf(", %.2f%% ok",
 364                                 100.0*(double)total.pass/(double)total.plan
 365                               );
 366                 }
 367                 printf("\n");
 368
 369         }
 370
 371         if (detail) {
 372                 printf("detail here\n");
 373         }
 374
 375         if (json) {
 376         printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n",
 377                         total.plan, total.ran, total.pass, total.fail
 378               );
 379         }
 380
 381 #if 0
 382         Files=8, Tests=508,  0 wallclock secs ( 0.08 usr +  0.01 sys =  0.09 CPU)
 383 Result: PASS
 384 #endif
 385
 386         return total.fail ? EXIT_FAILURE : EXIT_SUCCESS;
 387 }