pd.if.org Git - ctap/blob - prove.c

   1 #include <stdlib.h>
   2 #include <stdio.h>
   3 #include <string.h>
   4 #include <unistd.h>
   5 #include <errno.h>
   6 #include <sys/types.h>
   7 #include <sys/wait.h>
   8 #include <regex.h>
   9
  10 #define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) )
  11
  12 struct result {
  13         int test; /* i.e. the number of the test */
  14         char *name; /* name of the test, malloc */
  15         int pass; /* if the test passed */
  16         struct result *next; /* so we can make a linked list */
  17 };
  18
  19 struct testrun {
  20         char *name;
  21         int version;
  22         int plan;
  23         int ran;
  24         int pass;
  25         int fail;
  26         int skip;
  27         int todo, todo_pass, todo_fail;
  28         int bailout;
  29         struct result *results;
  30         struct result *last; /* last result, so we append rather than push */
  31         int interactive;
  32 };
  33
  34 int lookfor(char *needle, char *haystack, size_t n) {
  35         return strncasecmp(needle, haystack, n) == 0;
  36 }
  37
  38 int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) {
  39         return regexec(preg, string, 4, pmatch, 0) == 0;
  40 }
  41
  42 /* write n backspaces to stdout */
  43 void backup(int n) {
  44         while (n--) putchar('\b');
  45 }
  46
  47 void spaces(int n) {
  48         while (n--) putchar(' ');
  49 }
  50
  51 int runone(struct testrun *run) {
  52         int pipefd[2];
  53         pid_t cpid;
  54         FILE *tap;
  55         char *line;
  56         size_t nread, len = 0;
  57         int written = 0;
  58
  59         if (pipe(pipefd) == -1) {
  60                 perror("pipe");
  61                 exit(EXIT_FAILURE);
  62         }
  63
  64         cpid = fork();
  65         if (cpid == -1) {
  66                 perror("fork");
  67                 exit(EXIT_FAILURE);
  68         }
  69
  70         if (cpid == 0) {
  71                 /* child */
  72                 close(pipefd[0]); /* close read end */
  73                 close(1); /* close stdout */
  74                 dup(pipefd[1]); /* set stdout to the write end of the pipe */
  75                 execlp(run->name, run->name, NULL);
  76                 /* only gets here if exec failed */
  77                 printf("Bail Out! exec %s failed: %s\n", run->name, strerror(errno));
  78                 exit(EXIT_FAILURE);
  79         }
  80
  81         close(pipefd[1]);
  82         /* parent continue on */
  83         tap = fdopen(pipefd[0], "r");
  84
  85         regex_t plan, diagnostic, bail, test, directive, version;
  86         regmatch_t match[4];
  87
  88         regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED);
  89         regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED);
  90         regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED);
  91         regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED);
  92
  93         regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED);
  94         regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?",
  95                         REG_EXTENDED|REG_ICASE);
  96
  97
  98                 /* description: ok \d+([^#]+) */
  99                 /* directive: #.+ */
 100                 /* todo: # todo */
 101                 /* skip: # skip */
 102
 103                 /* bail out: ^Bail out! */
 104                 /* diagnostic: ^#.+ */
 105                 /* start of yaml: ^\s+--- */
 106                 /* end of yaml: ^\s+... */
 107                 /* between yaml lines must be indented */
 108                 /* anything else is an error */
 109
 110         while ((nread = getline(&line, &len, tap)) != -1) {
 111                 /* version: ^TAP version \d+ */
 112
 113                 /* plan: ^\d+\.\.\d+ */
 114                 if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) {
 115                         run->plan = atoi(line+match[1].rm_so);
 116                         /* TODO plans can have a diag skip reason */
 117                 } else
 118
 119                 /* ok: ^ok \d+ */
 120                 /* not ok: ^not ok \d+ */
 121                 if (rmatch(&test, line, match)) {
 122                         int test;
 123                         int todo = 0;
 124                         int skip = 0;
 125                         int pass = 0;
 126
 127                         run->ran++;
 128
 129                         if (run->interactive) {
 130                                 backup(written);
 131                                 written = printf("%d/%d", run->ran, run->plan);
 132                                 fflush(stdout);
 133                         }
 134
 135
 136                         /* if the "not" match fails, the test passed */
 137                         pass = match[1].rm_so == -1;
 138
 139                         /* test number */
 140                         test = atoi(line+match[3].rm_so);
 141
 142                         /* directive */
 143                         if (rmatch(&directive, line, match)) {
 144                                 todo = lookfor("todo", line+match[1].rm_so, 4);
 145                                 skip = lookfor("skip", line+match[1].rm_so, 4);
 146                         }
 147
 148                         /* check for diagnostics */
 149                         if (test != run->ran) {
 150                                 fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test);
 151                         }
 152
 153                         /* pass, todo, or skip are all pass */
 154                         if (pass || todo || skip) {
 155                                 run->pass++;
 156                         } else {
 157                                 run->fail++;
 158                         }
 159
 160                         if (todo && pass) {
 161                                 run->todo_pass++;
 162                         }
 163
 164                         if (skip) {
 165                                 run->skip++;
 166                         }
 167
 168                         if (!pass && !todo) {
 169                                 struct result *r;
 170                                 r = malloc(sizeof *r);
 171
 172                                 r->test = run->ran;
 173                                 r->pass = 0;
 174                                 if (run->last) {
 175                                         run->last->next = r;
 176                                 }
 177                                 if (!run->results) {
 178                                         run->results = r;
 179                                 }
 180                                 run->last = r;
 181                                 /* push test number onto fail list */
 182                         }
 183
 184                 }
 185                 else if (rmatch(&diagnostic, line, match)) {
 186
 187                 }
 188                 else if (rmatch(&version, line, match)) {
 189                         run->version = atoi(line + match[1].rm_so);
 190                 }
 191                 else if (rmatch(&bail, line, match)) {
 192                         run->bailout = 1;
 193                 }
 194                 else {
 195                         printf("Unknown line: %s", line);
 196                 }
 197
 198                 /* not ok: ^not ok \d+ */
 199                 /* description: ok \d+([^#]+) */
 200                 /* directive: #.+ */
 201                 /* todo: # todo */
 202                 /* skip: # skip */
 203
 204                 /* bail out: ^Bail out! */
 205                 /* diagnostic: ^#.+ */
 206                 /* start of yaml: ^\s+--- */
 207                 /* end of yaml: ^\s+... */
 208                 /* between yaml lines must be indented */
 209                 /* anything else is an error */
 210
 211         }
 212
 213         regfree(&plan);
 214         regfree(&test);
 215
 216         free(line);
 217         fclose(tap);
 218         wait(NULL);
 219
 220         if (run->ran < run->plan) {
 221                 run->fail += run->plan - run->ran;
 222         }
 223
 224         if (run->interactive) {
 225                 backup(written);
 226                 spaces(written);
 227                 backup(written);
 228                 fflush(stdout);
 229         }
 230
 231         return 0;
 232 }
 233
 234 void json(struct testrun *total, struct testrun *runs, int nruns) {
 235
 236 }
 237
 238 int main(int ac, char *av[]) {
 239         int maxnamelen = 0;
 240         ssize_t len;
 241         int i, j;
 242         struct testrun total = { 0 };
 243         struct testrun run = { 0 };
 244         struct testrun clear = { 0 };
 245         struct testrun *runs;
 246         int opt;
 247
 248         /*
 249          * -j json output
 250          * -p progress output
 251          * -i interactive, regardless of tty
 252          * -s summary text
 253          * -d detail text
 254          */
 255
 256         int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0;
 257
 258         interactive = isatty(1);
 259
 260         /* TODO -jj turn off everything else? */
 261         /* TODO -cC color escapes */
 262         while ((opt = getopt(ac, av, "jJpPiIsSdD")) != -1) {
 263                 switch (opt) {
 264                         case 'j': json = 1; break;
 265                         case 'J': json = 0; break;
 266                         case 'p': progress = 1; break;
 267                         case 'P': progress = 0; break;
 268                         case 'i': interactive = 1; break;
 269                         case 'I': interactive = 0; break;
 270                         case 's': summary = 1; break;
 271                         case 'S': summary = 0; break;
 272                         case 'd': detail = 1; break;
 273                         case 'D': detail = 0; break;
 274                         default: /* '?' */
 275                                 fprintf(stderr, "Usage: %s [-jpisd] file...\n",
 276                                                 av[0]);
 277                                 exit(EXIT_FAILURE);
 278                 }
 279         }
 280
 281         runs = malloc(ac * sizeof *runs);
 282
 283         for (i=optind; i<ac; i++) {
 284                 len = strlen(av[i]);
 285                 maxnamelen = GREATER(len, maxnamelen);
 286         }
 287
 288         for (i=optind; i<ac; i++) {
 289                 run = clear;
 290                 run.name = av[i];
 291                 run.interactive = interactive;
 292
 293                 if (progress) {
 294                         printf("%s ", run.name);
 295                         len = strlen(run.name);
 296                         for (j=len;j<maxnamelen+2;j++) {
 297                                 putchar('.');
 298                         }
 299                         putchar(' ');
 300                         fflush(stdout);
 301                 }
 302
 303                 runone(&run);
 304                 total.plan += run.plan;
 305                 total.ran += run.ran;
 306                 total.pass += run.pass;
 307                 total.fail += run.fail;
 308                 total.skip += run.skip;
 309
 310                 if (progress) {
 311                         if (run.bailout) {
 312                                 printf("Bail out!\n");
 313                         } else {
 314                                 if (run.fail > 0 ) printf("not ");
 315                                 printf("ok\n");
 316                         }
 317                 }
 318
 319                 runs[i-optind] = run;
 320         }
 321
 322         if (summary) {
 323                 printf("ran: %d/%d, pass: %d, fail: %d, skip: %d",
 324                                 total.ran, total.plan, total.pass, total.fail,
 325                                 total.skip
 326                       );
 327                 if (total.plan > 0) {
 328                         printf(", %.2f%% ok",
 329                                 100.0*(double)total.pass/(double)total.plan
 330                               );
 331                 }
 332                 printf("\n");
 333
 334         }
 335
 336         if (detail) {
 337                 printf("detail here\n");
 338         }
 339
 340         if (json) {
 341         printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n",
 342                         total.plan, total.ran, total.pass, total.fail
 343               );
 344         }
 345
 346 #if 0
 347         Files=8, Tests=508,  0 wallclock secs ( 0.08 usr +  0.01 sys =  0.09 CPU)
 348 Result: PASS
 349 #endif
 350
 351         return total.fail ? EXIT_FAILURE : EXIT_SUCCESS;
 352 }