--- /dev/null
+#ifndef TOK_H_
+#define TOK_H_ 1
+#include <stdio.h>
+/* START HEADER */
+struct token {
+ int type;
+ int rtype; /* a "real type", e.g. In, NAME, etc. */
+ int length;
+ int delimiter;
+ int nondigits;
+ int nonnames;
+ struct token *next;
+ char text[2048];
+};
+
+struct token_state {
+ FILE *input;
+ struct token *tok;
+
+ char *strinput;
+ int is_string;
+ int strlen;
+ int cursor;
+
+ int pc; /* previous char */
+ int cc; /* current char */
+ int nc; /* next char */
+ int push_back; /* true if next char is valid */
+
+ int prev_op; /* previous character was used as part of an operator */
+ int prev_word; /* previous character is part of a word */
+ int quoting; /* type of quoting */
+
+ char buf[2048];
+ int buflen;
+};
+
+struct token_state *ts_init(struct token_state *ts, FILE *f);
+int get_token(struct token_state *ts, struct token *tok);
+
+/* END HEADER */
+#endif