From: Nathan Wagner Date: Sun, 17 Jun 2018 14:18:17 +0000 (+0000) Subject: add prove test harness command X-Git-Url: https://pd.if.org/git/?p=ctap;a=commitdiff_plain;h=0e1d3c29ca96ab38343972209ed6e8c73719bb03 add prove test harness command --- diff --git a/Makefile b/Makefile index 46d37b0..3408d8c 100644 --- a/Makefile +++ b/Makefile @@ -7,5 +7,5 @@ main: $(OBJS) clean: rm -rf *.o main -test: main - @prove --exec '' ./main +test: main prove + ./prove ./main diff --git a/prove.c b/prove.c new file mode 100644 index 0000000..a0931c4 --- /dev/null +++ b/prove.c @@ -0,0 +1,334 @@ +#include +#include +#include +#include +#include +#include +#include + +#define GREATER(x,y) ( ((x) > (y)) ? (x) : (y) ) + +struct result { + int test; /* i.e. the number of the test */ + char *name; /* name of the test, malloc */ + int pass; /* if the test passed */ + struct result *next; /* so we can make a linked list */ +}; + +struct testrun { + char *name; + int version; + int plan; + int ran; + int pass; + int fail; + int skip; + int todo, todo_pass, todo_fail; + int bailout; + struct result *results; + struct result *last; /* last result, so we append rather than push */ + int interactive; +}; + +int lookfor(char *needle, char *haystack, size_t n) { + return strncasecmp(needle, haystack, n) == 0; +} + +int rmatch(const regex_t *preg, const char *string, regmatch_t pmatch[]) { + return regexec(preg, string, 4, pmatch, 0) == 0; +} + +/* write n backspaces to stdout */ +void backup(int n) { + while (n--) putchar('\b'); +} + +void spaces(int n) { + while (n--) putchar(' '); +} + +int runone(struct testrun *run) { + int pipefd[2]; + pid_t cpid; + FILE *tap; + char *line; + size_t nread, len = 0; + int written = 0; + + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + + if (cpid == 0) { + /* child */ + close(pipefd[0]); /* close read end */ + close(1); /* close stdout */ + dup(pipefd[1]); /* set stdout to the write end of the pipe */ + execlp(run->name, run->name, NULL); + } + + close(pipefd[1]); + /* parent continue on */ + tap = fdopen(pipefd[0], "r"); + + regex_t plan, diagnostic, bail, test, directive, version; + regmatch_t match[4]; + + regcomp(&plan, "^[[:digit:]]+\\.\\.([[:digit:]]+)", REG_EXTENDED); + regcomp(&diagnostic, "#[[:space:]]*(.+)", REG_EXTENDED); + regcomp(&bail, "^Bail Out![[:space:]]*(.+)", REG_EXTENDED); + regcomp(&version, "^TAP Version ([[:digit:]]+)", REG_EXTENDED); + + regcomp(&test, "^(not )?ok([[:space:]]*([[:digit:]]+))?", REG_EXTENDED); + regcomp(&directive, "#[[:space:]]*([^[:space:]]+)([[:space:]]+(.+))?", + REG_EXTENDED|REG_ICASE); + + + /* description: ok \d+([^#]+) */ + /* directive: #.+ */ + /* todo: # todo */ + /* skip: # skip */ + + /* bail out: ^Bail out! */ + /* diagnostic: ^#.+ */ + /* start of yaml: ^\s+--- */ + /* end of yaml: ^\s+... */ + /* between yaml lines must be indented */ + /* anything else is an error */ + + while ((nread = getline(&line, &len, tap)) != -1) { + /* version: ^TAP version \d+ */ + + /* plan: ^\d+\.\.\d+ */ + if (regexec(&plan, line, 2, match, 0) != REG_NOMATCH) { + run->plan = atoi(line+match[1].rm_so); + /* TODO plans can have a diag skip reason */ + } else + + /* ok: ^ok \d+ */ + /* not ok: ^not ok \d+ */ + if (rmatch(&test, line, match)) { + int test; + int todo = 0; + int skip = 0; + int pass = 0; + + /* if the "not" match fails, the test passed */ + pass = match[1].rm_so == -1; + + /* test number */ + test = atoi(line+match[3].rm_so); + + /* directive */ + if (rmatch(&directive, line, match)) { + todo = lookfor("todo", line+match[1].rm_so, 4); + skip = lookfor("skip", line+match[1].rm_so, 4); + } + + /* check for diagnostics */ + if (test != run->ran+1) { + fprintf(stderr, "expected test %d, got %d\n", run->ran+1, test); + } + + run->ran++; + + if (pass || todo) { + run->pass++; + } + + if (todo) { + if (pass) { + run->todo_pass++; + } + } + if (skip) { + run->skip++; + } + + if (!pass) { + struct result *r; + r = malloc(sizeof *r); + + r->test = run->ran; + r->pass = 0; + if (run->last) { + run->last->next = r; + } + if (!run->results) { + run->results = r; + } + run->last = r; + /* push test number onto fail list */ + } + + if (run->interactive) { + backup(written); + written = printf("%d/%d", run->ran, run->plan); + fflush(stdout); + } + } + else if (rmatch(&diagnostic, line, match)) { + + } + else if (rmatch(&version, line, match)) { + run->version = atoi(line + match[1].rm_so); + } + else if (rmatch(&bail, line, match)) { + run->bailout = 1; + } + else { + printf("Unknown line: %s", line); + } + + /* not ok: ^not ok \d+ */ + /* description: ok \d+([^#]+) */ + /* directive: #.+ */ + /* todo: # todo */ + /* skip: # skip */ + + /* bail out: ^Bail out! */ + /* diagnostic: ^#.+ */ + /* start of yaml: ^\s+--- */ + /* end of yaml: ^\s+... */ + /* between yaml lines must be indented */ + /* anything else is an error */ + + } + + regfree(&plan); + regfree(&test); + + free(line); + fclose(tap); + wait(NULL); + + if (run->ran < run->plan) { + run->fail += run->plan - run->ran; + } + + if (run->interactive) { + backup(written); + spaces(written); + backup(written); + fflush(stdout); + } + + return 0; +} + +void json(struct testrun *total, struct testrun *runs, int nruns) { + +} + +int main(int ac, char *av[]) { + int maxnamelen = 0; + ssize_t len; + int i, j; + struct testrun total = { 0 }; + struct testrun run = { 0 }; + struct testrun clear = { 0 }; + struct testrun *runs; + int opt; + + /* + * -j json output + * -p progress output + * -i interactive, regardless of tty + * -s summary text + * -d detail text + */ + + int interactive = 0, json = 0, summary = 1, progress = 1, detail = 0; + + interactive = isatty(1); + + /* TODO -jj turn off everything else? */ + /* TODO -cC color escapes */ + while ((opt = getopt(ac, av, "jJpPiIsSdD")) != -1) { + switch (opt) { + case 'j': json = 1; break; + case 'J': json = 0; break; + case 'p': progress = 1; break; + case 'P': progress = 0; break; + case 'i': interactive = 1; break; + case 'I': interactive = 0; break; + case 's': summary = 1; break; + case 'S': summary = 0; break; + case 'd': detail = 1; break; + case 'D': detail = 0; break; + default: /* '?' */ + fprintf(stderr, "Usage: %s [-jpisd] file...\n", + av[0]); + exit(EXIT_FAILURE); + } + } + + runs = malloc(ac * sizeof *runs); + + for (i=optind; i 0 ) printf("not "); + printf("ok\n"); + } + + runs[i-optind] = run; + } + + if (summary) { + printf("ran: %d/%d, pass: %d, fail: %d, %.2f%% ok\n", + total.ran, total.plan, total.pass, total.fail, + 100.0*(double)total.pass/(double)total.plan + ); + + } + + if (detail) { + printf("detail here\n"); + } + + if (json) { + printf("{ \"plan\": %d, \"ran\": %d, \"pass\": %d, \"fail\": %d }\n", + total.plan, total.ran, total.pass, total.fail + ); + } + +#if 0 + Files=8, Tests=508, 0 wallclock secs ( 0.08 usr + 0.01 sys = 0.09 CPU) +Result: PASS +#endif + + return total.fail ? EXIT_FAILURE : EXIT_SUCCESS; +}