pd.if.org Git - zpackage/blob - t/ctap/prove.c

   1 #define _POSIX_C_SOURCE 200809L
   2
   3 #include <stdlib.h>
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <strings.h>
   7 #include <unistd.h>
   8 #include <errno.h>
   9 #include <sys/types.h>
  10 #include <sys/wait.h>
  11 #include <regex.h>
  12
  13 #define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) )
  14
  15 struct result {
  16         int test; /* i.e. the number of the test */
  17         char *name; /* name of the test, malloc */
  18         int pass; /* if the test passed */
  19         struct result *next; /* so we can make a linked list */
  20 };
  21
  22 struct testrun {
  23         char *name;
  24         int readstdin;
  25         int version;
  26         int plan;
  27         int ran;
  28         int pass;
  29         int fail;
  30         int skip;
  31         int todo, todo_pass, todo_fail;
  32         int bailout;
  33         struct result *results;
  34         struct result *last; /* last result, so we append rather than push */
  35         int interactive;
  36 };
  37
  38 int lookfor(char *needle, char *haystack, size_t n) {
  39         return strncasecmp(needle, haystack, n) == 0;
  40 }
  41
  42 int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) {
  43         return regexec(preg, string, 4, pmatch, 0) == 0;
  44 }
  45
  46 /* write n backspaces to stdout */
  47 void backup(int n) {
  48         while (n--) putchar('\b');
  49 }
  50
  51 void spaces(int n) {
  52         while (n--) putchar(' ');
  53 }
  54
  55 int runone(struct testrun *run) {
  56         int pipefd[2];
  57         pid_t cpid;
  58         FILE *tap;
  59         char *line;
  60         ssize_t nread;
  61         size_t len = 0;
  62         int written = 0;
  63
  64         if (!run->readstdin) {
  65                 if (pipe(pipefd) == -1) {
  66                         perror("pipe");
  67                         exit(EXIT_FAILURE);
  68                 }
  69
  70                 cpid = fork();
  71                 if (cpid == -1) {
  72                         perror("fork");
  73                         exit(EXIT_FAILURE);
  74                 }
  75
  76                 if (cpid == 0) {
  77                         /* child */
  78                         close(pipefd[0]); /* close read end */
  79                         close(1); /* close stdout */
  80                         dup(pipefd[1]); /* set stdout to the write end of the pipe */
  81                         execlp(run->name, run->name, NULL);
  82                         /* only gets here if exec failed */
  83                         printf("Bail Out! exec %s failed: %s\n", run->name, strerror(errno));
  84                         exit(EXIT_FAILURE);
  85                 }
  86
  87                 close(pipefd[1]);
  88                 /* parent continue on */
  89                 tap = fdopen(pipefd[0], "r");
  90         } else {
  91                 tap = stdin;
  92         }
  93
  94         regex_t plan, diagnostic, bail, test, directive, version;
  95         regmatch_t match[4];
  96
  97         regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED);
  98         regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED);
  99         regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED|REG_ICASE);
 100         regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED);
 101
 102         regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED);
 103         regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?",
 104                         REG_EXTENDED|REG_ICASE);
 105
 106
 107                 /* description: ok \d+([^#]+) */
 108                 /* directive: #.+ */
 109                 /* todo: # todo */
 110                 /* skip: # skip */
 111
 112                 /* bail out: ^Bail out! */
 113                 /* diagnostic: ^#.+ */
 114                 /* start of yaml: ^\s+--- */
 115                 /* end of yaml: ^\s+... */
 116                 /* between yaml lines must be indented */
 117                 /* anything else is an error */
 118
 119         while ((nread = getline(&line, &len, tap)) != -1) {
 120                 /* version: ^TAP version \d+ */
 121
 122                 /* plan: ^\d+\.\.\d+ */
 123                 if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) {
 124                         run->plan = atoi(line+match[1].rm_so);
 125                         /* TODO plans can have a diag skip reason */
 126                 } else
 127
 128                 /* ok: ^ok \d+ */
 129                 /* not ok: ^not ok \d+ */
 130                 if (rmatch(&test, line, match)) {
 131                         int test;
 132                         int todo = 0;
 133                         int skip = 0;
 134                         int pass = 0;
 135
 136                         run->ran++;
 137
 138                         if (run->interactive) {
 139                                 backup(written);
 140                                 written = printf("%d/%d", run->ran, run->plan);
 141                                 fflush(stdout);
 142                         }
 143
 144
 145                         /* if the "not" match fails, the test passed */
 146                         pass = match[1].rm_so == -1;
 147
 148                         /* test number */
 149                         test = atoi(line+match[3].rm_so);
 150
 151                         /* directive */
 152                         if (rmatch(&directive, line, match)) {
 153                                 todo = lookfor("todo", line+match[1].rm_so, 4);
 154                                 skip = lookfor("skip", line+match[1].rm_so, 4);
 155                         }
 156
 157                         /* check for diagnostics */
 158                         if (test != run->ran) {
 159                                 fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test);
 160                         }
 161
 162                         /* pass, todo, or skip are all pass */
 163                         if (pass || todo || skip) {
 164                                 run->pass++;
 165                         } else {
 166                                 run->fail++;
 167                         }
 168
 169                         if (todo && pass) {
 170                                 run->todo_pass++;
 171                         }
 172
 173                         if (skip) {
 174                                 run->skip++;
 175                         }
 176
 177                         if (!pass && !todo) {
 178                                 struct result *r;
 179                                 r = malloc(sizeof *r);
 180
 181                                 r->test = run->ran;
 182                                 r->pass = 0;
 183                                 if (run->last) {
 184                                         run->last->next = r;
 185                                 }
 186                                 if (!run->results) {
 187                                         run->results = r;
 188                                 }
 189                                 run->last = r;
 190                                 /* push test number onto fail list */
 191                         }
 192
 193                 }
 194                 else if (rmatch(&diagnostic, line, match)) {
 195
 196                 }
 197                 else if (rmatch(&version, line, match)) {
 198                         run->version = atoi(line + match[1].rm_so);
 199                 }
 200                 else if (rmatch(&bail, line, match)) {
 201                         run->bailout = 1;
 202                 }
 203                 else {
 204                         printf("Unknown line: %s", line);
 205                 }
 206
 207                 /* not ok: ^not ok \d+ */
 208                 /* description: ok \d+([^#]+) */
 209                 /* directive: #.+ */
 210                 /* todo: # todo */
 211                 /* skip: # skip */
 212
 213                 /* bail out: ^Bail out! */
 214                 /* diagnostic: ^#.+ */
 215                 /* start of yaml: ^\s+--- */
 216                 /* end of yaml: ^\s+... */
 217                 /* between yaml lines must be indented */
 218                 /* anything else is an error */
 219
 220         }
 221
 222         regfree(&plan);
 223         regfree(&test);
 224
 225         free(line);
 226         if (!run->readstdin) {
 227                 fclose(tap);
 228         }
 229         wait(NULL);
 230
 231         if (run->ran < run->plan) {
 232                 run->fail += run->plan - run->ran;
 233         }
 234
 235         if (run->interactive) {
 236                 backup(written);
 237                 spaces(written);
 238                 backup(written);
 239                 fflush(stdout);
 240         }
 241
 242         return 0;
 243 }
 244
 245 void json(struct testrun *total, struct testrun *runs, int nruns) {
 246         if (total && runs && nruns) {
 247                 return;
 248         }
 249         return;
 250 }
 251
 252 int main(int ac, char *av[]) {
 253         int maxnamelen = 0;
 254         ssize_t len;
 255         int i, j;
 256         struct testrun total = { 0 };
 257         struct testrun run = { 0 };
 258         struct testrun clear = { 0 };
 259         struct testrun *runs;
 260         int opt;
 261         char *stdinname = "<stdin>";
 262         int stdinnamelen = 0;
 263
 264         /*
 265          * -j json output
 266          * -p progress output
 267          * -i interactive, regardless of tty
 268          * -s summary text
 269          * -d detail text
 270          */
 271
 272         int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0;
 273
 274         interactive = isatty(1);
 275
 276         /* TODO -jj turn off everything else? */
 277         /* TODO -cC color escapes */
 278         while ((opt = getopt(ac, av, "jJpPiIsSdDn:")) != -1) {
 279                 switch (opt) {
 280                         case 'j': json = 1; break;
 281                         case 'J': json = 0; break;
 282                         case 'p': progress = 1; break;
 283                         case 'P': progress = 0; break;
 284                         case 'i': interactive = 1; break;
 285                         case 'I': interactive = 0; break;
 286                         case 's': summary = 1; break;
 287                         case 'S': summary = 0; break;
 288                         case 'd': detail = 1; break;
 289                         case 'D': detail = 0; break;
 290                         case 'n': stdinname = optarg;
 291                                   break;
 292                         default: /* '?' */
 293                                 fprintf(stderr, "Usage: %s [-jpisd] file...\n",
 294                                                 av[0]);
 295                                 exit(EXIT_FAILURE);
 296                 }
 297         }
 298         stdinnamelen = strlen(stdinname);
 299
 300         runs = malloc(ac * sizeof *runs);
 301
 302         for (i=optind; i<ac; i++) {
 303                 if (!strcmp(av[i], "-")) {
 304                         maxnamelen = GREATER(stdinnamelen, maxnamelen);
 305                 } else {
 306                         len = strlen(av[i]);
 307                         maxnamelen = GREATER(len, maxnamelen);
 308                 }
 309         }
 310
 311         for (i=optind; i<ac; i++) {
 312                 run = clear;
 313                 run.interactive = interactive;
 314
 315                 if (!strcmp(av[i], "-")) {
 316                         run.readstdin = 1;
 317                         run.name = stdinname;
 318                 } else {
 319                         run.readstdin = 0;
 320                         run.name = av[i];
 321                 }
 322
 323                 if (progress) {
 324                         printf("%s ", run.name);
 325                         len = strlen(run.name);
 326                         for (j=len;j<maxnamelen+2;j++) {
 327                                 putchar('.');
 328                         }
 329                         putchar(' ');
 330                         fflush(stdout);
 331                 }
 332
 333                 runone(&run);
 334                 total.plan += run.plan;
 335                 total.ran += run.ran;
 336                 total.pass += run.pass;
 337                 total.fail += run.fail;
 338                 total.skip += run.skip;
 339
 340                 if (progress) {
 341                         if (run.bailout) {
 342                                 printf("Bail out!\n");
 343                         } else {
 344                                 if (run.fail > 0 ) printf("not ");
 345                                 printf("ok\n");
 346                         }
 347                 }
 348
 349                 runs[i-optind] = run;
 350         }
 351
 352         if (summary) {
 353                 printf("ran: %d/%d, pass: %d, fail: %d, skip: %d",
 354                                 total.ran, total.plan, total.pass, total.fail,
 355                                 total.skip
 356                       );
 357                 if (total.plan > 0) {
 358                         printf(", %.2f%% ok",
 359                                 100.0*(double)total.pass/(double)total.plan
 360                               );
 361                 }
 362                 printf("\n");
 363
 364         }
 365
 366         if (detail) {
 367                 printf("detail here\n");
 368         }
 369
 370         if (json) {
 371         printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n",
 372                         total.plan, total.ran, total.pass, total.fail
 373               );
 374         }
 375
 376 #if 0
 377         Files=8, Tests=508,  0 wallclock secs ( 0.08 usr +  0.01 sys =  0.09 CPU)
 378 Result: PASS
 379 #endif
 380
 381         return total.fail ? EXIT_FAILURE : EXIT_SUCCESS;
 382 }