pd.if.org Git - ctap/blob - prove.c

   1 #define _POSIX_C_SOURCE 200809L
   2
   3 #include <stdlib.h>
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <strings.h>
   7 #include <unistd.h>
   8 #include <errno.h>
   9 #include <sys/types.h>
  10 #include <sys/wait.h>
  11 #include <regex.h>
  12
  13 #define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) )
  14
  15 struct result {
  16         int test; /* i.e. the number of the test */
  17         char *name; /* name of the test, malloc */
  18         int pass; /* if the test passed */
  19         struct result *next; /* so we can make a linked list */
  20 };
  21
  22 struct testrun {
  23         char *name;
  24         int version;
  25         int plan;
  26         int ran;
  27         int pass;
  28         int fail;
  29         int skip;
  30         int todo, todo_pass, todo_fail;
  31         int bailout;
  32         struct result *results;
  33         struct result *last; /* last result, so we append rather than push */
  34         int interactive;
  35 };
  36
  37 int lookfor(char *needle, char *haystack, size_t n) {
  38         return strncasecmp(needle, haystack, n) == 0;
  39 }
  40
  41 int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) {
  42         return regexec(preg, string, 4, pmatch, 0) == 0;
  43 }
  44
  45 /* write n backspaces to stdout */
  46 void backup(int n) {
  47         while (n--) putchar('\b');
  48 }
  49
  50 void spaces(int n) {
  51         while (n--) putchar(' ');
  52 }
  53
  54 int runone(struct testrun *run) {
  55         int pipefd[2];
  56         pid_t cpid;
  57         FILE *tap;
  58         char *line;
  59         ssize_t nread;
  60         size_t len = 0;
  61         int written = 0;
  62
  63         if (pipe(pipefd) == -1) {
  64                 perror("pipe");
  65                 exit(EXIT_FAILURE);
  66         }
  67
  68         cpid = fork();
  69         if (cpid == -1) {
  70                 perror("fork");
  71                 exit(EXIT_FAILURE);
  72         }
  73
  74         if (cpid == 0) {
  75                 /* child */
  76                 close(pipefd[0]); /* close read end */
  77                 close(1); /* close stdout */
  78                 dup(pipefd[1]); /* set stdout to the write end of the pipe */
  79                 execlp(run->name, run->name, NULL);
  80                 /* only gets here if exec failed */
  81                 printf("Bail Out! exec %s failed: %s\n", run->name, strerror(errno));
  82                 exit(EXIT_FAILURE);
  83         }
  84
  85         close(pipefd[1]);
  86         /* parent continue on */
  87         tap = fdopen(pipefd[0], "r");
  88
  89         regex_t plan, diagnostic, bail, test, directive, version;
  90         regmatch_t match[4];
  91
  92         regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED);
  93         regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED);
  94         regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED);
  95         regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED);
  96
  97         regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED);
  98         regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?",
  99                         REG_EXTENDED|REG_ICASE);
 100
 101
 102                 /* description: ok \d+([^#]+) */
 103                 /* directive: #.+ */
 104                 /* todo: # todo */
 105                 /* skip: # skip */
 106
 107                 /* bail out: ^Bail out! */
 108                 /* diagnostic: ^#.+ */
 109                 /* start of yaml: ^\s+--- */
 110                 /* end of yaml: ^\s+... */
 111                 /* between yaml lines must be indented */
 112                 /* anything else is an error */
 113
 114         while ((nread = getline(&line, &len, tap)) != -1) {
 115                 /* version: ^TAP version \d+ */
 116
 117                 /* plan: ^\d+\.\.\d+ */
 118                 if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) {
 119                         run->plan = atoi(line+match[1].rm_so);
 120                         /* TODO plans can have a diag skip reason */
 121                 } else
 122
 123                 /* ok: ^ok \d+ */
 124                 /* not ok: ^not ok \d+ */
 125                 if (rmatch(&test, line, match)) {
 126                         int test;
 127                         int todo = 0;
 128                         int skip = 0;
 129                         int pass = 0;
 130
 131                         run->ran++;
 132
 133                         if (run->interactive) {
 134                                 backup(written);
 135                                 written = printf("%d/%d", run->ran, run->plan);
 136                                 fflush(stdout);
 137                         }
 138
 139
 140                         /* if the "not" match fails, the test passed */
 141                         pass = match[1].rm_so == -1;
 142
 143                         /* test number */
 144                         test = atoi(line+match[3].rm_so);
 145
 146                         /* directive */
 147                         if (rmatch(&directive, line, match)) {
 148                                 todo = lookfor("todo", line+match[1].rm_so, 4);
 149                                 skip = lookfor("skip", line+match[1].rm_so, 4);
 150                         }
 151
 152                         /* check for diagnostics */
 153                         if (test != run->ran) {
 154                                 fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test);
 155                         }
 156
 157                         /* pass, todo, or skip are all pass */
 158                         if (pass || todo || skip) {
 159                                 run->pass++;
 160                         } else {
 161                                 run->fail++;
 162                         }
 163
 164                         if (todo && pass) {
 165                                 run->todo_pass++;
 166                         }
 167
 168                         if (skip) {
 169                                 run->skip++;
 170                         }
 171
 172                         if (!pass && !todo) {
 173                                 struct result *r;
 174                                 r = malloc(sizeof *r);
 175
 176                                 r->test = run->ran;
 177                                 r->pass = 0;
 178                                 if (run->last) {
 179                                         run->last->next = r;
 180                                 }
 181                                 if (!run->results) {
 182                                         run->results = r;
 183                                 }
 184                                 run->last = r;
 185                                 /* push test number onto fail list */
 186                         }
 187
 188                 }
 189                 else if (rmatch(&diagnostic, line, match)) {
 190
 191                 }
 192                 else if (rmatch(&version, line, match)) {
 193                         run->version = atoi(line + match[1].rm_so);
 194                 }
 195                 else if (rmatch(&bail, line, match)) {
 196                         run->bailout = 1;
 197                 }
 198                 else {
 199                         printf("Unknown line: %s", line);
 200                 }
 201
 202                 /* not ok: ^not ok \d+ */
 203                 /* description: ok \d+([^#]+) */
 204                 /* directive: #.+ */
 205                 /* todo: # todo */
 206                 /* skip: # skip */
 207
 208                 /* bail out: ^Bail out! */
 209                 /* diagnostic: ^#.+ */
 210                 /* start of yaml: ^\s+--- */
 211                 /* end of yaml: ^\s+... */
 212                 /* between yaml lines must be indented */
 213                 /* anything else is an error */
 214
 215         }
 216
 217         regfree(&plan);
 218         regfree(&test);
 219
 220         free(line);
 221         fclose(tap);
 222         wait(NULL);
 223
 224         if (run->ran < run->plan) {
 225                 run->fail += run->plan - run->ran;
 226         }
 227
 228         if (run->interactive) {
 229                 backup(written);
 230                 spaces(written);
 231                 backup(written);
 232                 fflush(stdout);
 233         }
 234
 235         return 0;
 236 }
 237
 238 void json(struct testrun *total, struct testrun *runs, int nruns) {
 239         if (total && runs && nruns) {
 240                 return;
 241         }
 242         return;
 243 }
 244
 245 int main(int ac, char *av[]) {
 246         int maxnamelen = 0;
 247         ssize_t len;
 248         int i, j;
 249         struct testrun total = { 0 };
 250         struct testrun run = { 0 };
 251         struct testrun clear = { 0 };
 252         struct testrun *runs;
 253         int opt;
 254
 255         /*
 256          * -j json output
 257          * -p progress output
 258          * -i interactive, regardless of tty
 259          * -s summary text
 260          * -d detail text
 261          */
 262
 263         int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0;
 264
 265         interactive = isatty(1);
 266
 267         /* TODO -jj turn off everything else? */
 268         /* TODO -cC color escapes */
 269         while ((opt = getopt(ac, av, "jJpPiIsSdD")) != -1) {
 270                 switch (opt) {
 271                         case 'j': json = 1; break;
 272                         case 'J': json = 0; break;
 273                         case 'p': progress = 1; break;
 274                         case 'P': progress = 0; break;
 275                         case 'i': interactive = 1; break;
 276                         case 'I': interactive = 0; break;
 277                         case 's': summary = 1; break;
 278                         case 'S': summary = 0; break;
 279                         case 'd': detail = 1; break;
 280                         case 'D': detail = 0; break;
 281                         default: /* '?' */
 282                                 fprintf(stderr, "Usage: %s [-jpisd] file...\n",
 283                                                 av[0]);
 284                                 exit(EXIT_FAILURE);
 285                 }
 286         }
 287
 288         runs = malloc(ac * sizeof *runs);
 289
 290         for (i=optind; i<ac; i++) {
 291                 len = strlen(av[i]);
 292                 maxnamelen = GREATER(len, maxnamelen);
 293         }
 294
 295         for (i=optind; i<ac; i++) {
 296                 run = clear;
 297                 run.name = av[i];
 298                 run.interactive = interactive;
 299
 300                 if (progress) {
 301                         printf("%s ", run.name);
 302                         len = strlen(run.name);
 303                         for (j=len;j<maxnamelen+2;j++) {
 304                                 putchar('.');
 305                         }
 306                         putchar(' ');
 307                         fflush(stdout);
 308                 }
 309
 310                 runone(&run);
 311                 total.plan += run.plan;
 312                 total.ran += run.ran;
 313                 total.pass += run.pass;
 314                 total.fail += run.fail;
 315                 total.skip += run.skip;
 316
 317                 if (progress) {
 318                         if (run.bailout) {
 319                                 printf("Bail out!\n");
 320                         } else {
 321                                 if (run.fail > 0 ) printf("not ");
 322                                 printf("ok\n");
 323                         }
 324                 }
 325
 326                 runs[i-optind] = run;
 327         }
 328
 329         if (summary) {
 330                 printf("ran: %d/%d, pass: %d, fail: %d, skip: %d",
 331                                 total.ran, total.plan, total.pass, total.fail,
 332                                 total.skip
 333                       );
 334                 if (total.plan > 0) {
 335                         printf(", %.2f%% ok",
 336                                 100.0*(double)total.pass/(double)total.plan
 337                               );
 338                 }
 339                 printf("\n");
 340
 341         }
 342
 343         if (detail) {
 344                 printf("detail here\n");
 345         }
 346
 347         if (json) {
 348         printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n",
 349                         total.plan, total.ran, total.pass, total.fail
 350               );
 351         }
 352
 353 #if 0
 354         Files=8, Tests=508,  0 wallclock secs ( 0.08 usr +  0.01 sys =  0.09 CPU)
 355 Result: PASS
 356 #endif
 357
 358         return total.fail ? EXIT_FAILURE : EXIT_SUCCESS;
 359 }