pd.if.org Git - ctap/blob - prove.c

   1 #include <stdlib.h>
   2 #include <stdio.h>
   3 #include <string.h>
   4 #include <unistd.h>
   5 #include <sys/types.h>
   6 #include <sys/wait.h>
   7 #include <regex.h>
   8
   9 #define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) )
  10
  11 struct result {
  12         int test; /* i.e. the number of the test */
  13         char *name; /* name of the test, malloc */
  14         int pass; /* if the test passed */
  15         struct result *next; /* so we can make a linked list */
  16 };
  17
  18 struct testrun {
  19         char *name;
  20         int version;
  21         int plan;
  22         int ran;
  23         int pass;
  24         int fail;
  25         int skip;
  26         int todo, todo_pass, todo_fail;
  27         int bailout;
  28         struct result *results;
  29         struct result *last; /* last result, so we append rather than push */
  30         int interactive;
  31 };
  32
  33 int lookfor(char *needle, char *haystack, size_t n) {
  34         return strncasecmp(needle, haystack, n) == 0;
  35 }
  36
  37 int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) {
  38         return regexec(preg, string, 4, pmatch, 0) == 0;
  39 }
  40
  41 /* write n backspaces to stdout */
  42 void backup(int n) {
  43         while (n--) putchar('\b');
  44 }
  45
  46 void spaces(int n) {
  47         while (n--) putchar(' ');
  48 }
  49
  50 int runone(struct testrun *run) {
  51         int pipefd[2];
  52         pid_t cpid;
  53         FILE *tap;
  54         char *line;
  55         size_t nread, len = 0;
  56         int written = 0;
  57
  58         if (pipe(pipefd) == -1) {
  59                 perror("pipe");
  60                 exit(EXIT_FAILURE);
  61         }
  62
  63         cpid = fork();
  64         if (cpid == -1) {
  65                 perror("fork");
  66                 exit(EXIT_FAILURE);
  67         }
  68
  69         if (cpid == 0) {
  70                 /* child */
  71                 close(pipefd[0]); /* close read end */
  72                 close(1); /* close stdout */
  73                 dup(pipefd[1]); /* set stdout to the write end of the pipe */
  74                 execlp(run->name, run->name, NULL);
  75         }
  76
  77         close(pipefd[1]);
  78         /* parent continue on */
  79         tap = fdopen(pipefd[0], "r");
  80
  81         regex_t plan, diagnostic, bail, test, directive, version;
  82         regmatch_t match[4];
  83
  84         regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED);
  85         regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED);
  86         regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED);
  87         regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED);
  88
  89         regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED);
  90         regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?",
  91                         REG_EXTENDED|REG_ICASE);
  92
  93
  94                 /* description: ok \d+([^#]+) */
  95                 /* directive: #.+ */
  96                 /* todo: # todo */
  97                 /* skip: # skip */
  98
  99                 /* bail out: ^Bail out! */
 100                 /* diagnostic: ^#.+ */
 101                 /* start of yaml: ^\s+--- */
 102                 /* end of yaml: ^\s+... */
 103                 /* between yaml lines must be indented */
 104                 /* anything else is an error */
 105
 106         while ((nread = getline(&line, &len, tap)) != -1) {
 107                 /* version: ^TAP version \d+ */
 108
 109                 /* plan: ^\d+\.\.\d+ */
 110                 if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) {
 111                         run->plan = atoi(line+match[1].rm_so);
 112                         /* TODO plans can have a diag skip reason */
 113                 } else
 114
 115                 /* ok: ^ok \d+ */
 116                 /* not ok: ^not ok \d+ */
 117                 if (rmatch(&test, line, match)) {
 118                         int test;
 119                         int todo = 0;
 120                         int skip = 0;
 121                         int pass = 0;
 122
 123                         /* if the "not" match fails, the test passed */
 124                         pass = match[1].rm_so == -1;
 125
 126                         /* test number */
 127                         test = atoi(line+match[3].rm_so);
 128
 129                         /* directive */
 130                         if (rmatch(&directive, line, match)) {
 131                                 todo = lookfor("todo", line+match[1].rm_so, 4);
 132                                 skip = lookfor("skip", line+match[1].rm_so, 4);
 133                         }
 134
 135                         /* check for diagnostics */
 136                         if (test != run->ran+1) {
 137                                 fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test);
 138                         }
 139
 140                         run->ran++;
 141
 142                         if (pass || todo) {
 143                                 run->pass++;
 144                         }
 145
 146                         if (todo) {
 147                                 if (pass) {
 148                                         run->todo_pass++;
 149                                 }
 150                         }
 151                         if (skip) {
 152                                 run->skip++;
 153                         }
 154
 155                         if (!pass) {
 156                                 struct result *r;
 157                                 r = malloc(sizeof *r);
 158
 159                                 r->test = run->ran;
 160                                 r->pass = 0;
 161                                 if (run->last) {
 162                                         run->last->next = r;
 163                                 }
 164                                 if (!run->results) {
 165                                         run->results = r;
 166                                 }
 167                                 run->last = r;
 168                                 /* push test number onto fail list */
 169                         }
 170
 171                         if (run->interactive) {
 172                                 backup(written);
 173                                 written = printf("%d/%d", run->ran, run->plan);
 174                                 fflush(stdout);
 175                         }
 176                 }
 177                 else if (rmatch(&diagnostic, line, match)) {
 178
 179                 }
 180                 else if (rmatch(&version, line, match)) {
 181                         run->version = atoi(line + match[1].rm_so);
 182                 }
 183                 else if (rmatch(&bail, line, match)) {
 184                         run->bailout = 1;
 185                 }
 186                 else {
 187                         printf("Unknown line: %s", line);
 188                 }
 189
 190                 /* not ok: ^not ok \d+ */
 191                 /* description: ok \d+([^#]+) */
 192                 /* directive: #.+ */
 193                 /* todo: # todo */
 194                 /* skip: # skip */
 195
 196                 /* bail out: ^Bail out! */
 197                 /* diagnostic: ^#.+ */
 198                 /* start of yaml: ^\s+--- */
 199                 /* end of yaml: ^\s+... */
 200                 /* between yaml lines must be indented */
 201                 /* anything else is an error */
 202
 203         }
 204
 205         regfree(&plan);
 206         regfree(&test);
 207
 208         free(line);
 209         fclose(tap);
 210         wait(NULL);
 211
 212         if (run->ran < run->plan) {
 213                 run->fail += run->plan - run->ran;
 214         }
 215
 216         if (run->interactive) {
 217                 backup(written);
 218                 spaces(written);
 219                 backup(written);
 220                 fflush(stdout);
 221         }
 222
 223         return 0;
 224 }
 225
 226 void json(struct testrun *total, struct testrun *runs, int nruns) {
 227
 228 }
 229
 230 int main(int ac, char *av[]) {
 231         int maxnamelen = 0;
 232         ssize_t len;
 233         int i, j;
 234         struct testrun total = { 0 };
 235         struct testrun run = { 0 };
 236         struct testrun clear = { 0 };
 237         struct testrun *runs;
 238         int opt;
 239
 240         /*
 241          * -j json output
 242          * -p progress output
 243          * -i interactive, regardless of tty
 244          * -s summary text
 245          * -d detail text
 246          */
 247
 248         int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0;
 249
 250         interactive = isatty(1);
 251
 252         /* TODO -jj turn off everything else? */
 253         /* TODO -cC color escapes */
 254         while ((opt = getopt(ac, av, "jJpPiIsSdD")) != -1) {
 255                 switch (opt) {
 256                         case 'j': json = 1; break;
 257                         case 'J': json = 0; break;
 258                         case 'p': progress = 1; break;
 259                         case 'P': progress = 0; break;
 260                         case 'i': interactive = 1; break;
 261                         case 'I': interactive = 0; break;
 262                         case 's': summary = 1; break;
 263                         case 'S': summary = 0; break;
 264                         case 'd': detail = 1; break;
 265                         case 'D': detail = 0; break;
 266                         default: /* '?' */
 267                                 fprintf(stderr, "Usage: %s [-jpisd] file...\n",
 268                                                 av[0]);
 269                                 exit(EXIT_FAILURE);
 270                 }
 271         }
 272
 273         runs = malloc(ac * sizeof *runs);
 274
 275         for (i=optind; i<ac; i++) {
 276                 len = strlen(av[i]);
 277                 maxnamelen = GREATER(len, maxnamelen);
 278         }
 279
 280         for (i=optind; i<ac; i++) {
 281                 run = clear;
 282                 run.name = av[i];
 283                 run.plan = -1;
 284                 run.interactive = interactive;
 285
 286                 if (progress) {
 287                         printf("%s ", run.name);
 288                         len = strlen(run.name);
 289                         for (j=len;j<maxnamelen+2;j++) {
 290                                 putchar('.');
 291                         }
 292                         putchar(' ');
 293                         fflush(stdout);
 294                 }
 295
 296                 runone(&run);
 297                 total.plan += run.plan;
 298                 total.ran += run.ran;
 299                 total.pass += run.pass;
 300                 total.fail += run.fail;
 301
 302                 if (progress) {
 303                         if (run.fail > 0 ) printf("not ");
 304                         printf("ok\n");
 305                 }
 306
 307                 runs[i-optind] = run;
 308         }
 309
 310         if (summary) {
 311                 printf("ran: %d/%d, pass: %d, fail: %d, %.2f%% ok\n",
 312                                 total.ran, total.plan, total.pass, total.fail,
 313                                 100.0*(double)total.pass/(double)total.plan
 314                       );
 315
 316         }
 317
 318         if (detail) {
 319                 printf("detail here\n");
 320         }
 321
 322         if (json) {
 323         printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n",
 324                         total.plan, total.ran, total.pass, total.fail
 325               );
 326         }
 327
 328 #if 0
 329         Files=8, Tests=508,  0 wallclock secs ( 0.08 usr +  0.01 sys =  0.09 CPU)
 330 Result: PASS
 331 #endif
 332
 333         return total.fail ? EXIT_FAILURE : EXIT_SUCCESS;
 334 }