+++ /dev/null
-#include <stdio.h>
-#include "gram.h"
-
-/* START HEADER */
-struct token {
- int type;
- int rtype; /* a "real type", e.g. In, NAME, etc. */
- int length;
- int delimiter;
- int nondigits;
- int nonnames;
- struct token *next;
- char text[2048];
-};
-
-struct token_state {
- FILE *input;
- struct token *tok;
-
- char *strinput;
- int is_string;
- int strlen;
- int cursor;
-
- int pc; /* previous char */
- int cc; /* current char */
- int nc; /* next char */
- int push_back; /* true if next char is valid */
-
- int prev_op; /* previous character was used as part of an operator */
- int prev_word; /* previous character is part of a word */
- int quoting; /* type of quoting */
-
- char buf[2048];
- int buflen;
-};
-
-struct token_state *ts_init(struct token_state *ts, FILE *f);
-int get_token(struct token_state *ts, struct token *tok);
-
-/* END HEADER */
-
-#if DEBUG_TOKEN
-static void ptoken(struct token_state *ts) {
- fprintf(stderr, "token %d = %s\n", ts->tok->type, ts->tok->text);
-}
-#endif
-
-static int nextch(struct token_state *ts);
-static int peek(struct token_state *ts);
-static int nextchar(struct token_state *ts);
-static int process_char(struct token_state *ts);
-
-static int peek(struct token_state *ts) {
- if (!ts->push_back) {
- ts->nc = nextchar(ts);
- ts->push_back = 1;
- }
- return ts->nc;
-}
-
-static int nextch(struct token_state *ts) {
- if (!ts->push_back) {
- ts->nc = nextchar(ts);
- }
- ts->pc = ts->cc;
- ts->cc = ts->nc;
- ts->push_back = 0;
- return ts->cc;
-}
-
-#if 0
-static int pushback(struct token_state *ts) {
- /* TODO error if already pushed back? */
- ts->nc = ts->cc;
- ts->cc = ts->pc;
- return ts->cc;
-}
-#endif
-
-static int add_to_token(struct token_state *ts, int ch) {
- ts->tok->text[ts->tok->length++] = (char)ch;
-#if DEBUG_TOKEN
- ptoken(ts);
-#endif
- return ts->tok->length;
-}
-
-static int start_token(struct token_state *ts, int type, int ch) {
- int i;
- for (i=0;i<sizeof ts->tok->text; i++) {
- ts->tok->text[i] = 0;
- }
- ts->tok->type = type;
- ts->tok->text[0] = (char)ch;
- ts->tok->length = 1;
-#if DEBUG_TOKEN
- fprintf(stderr, "starting ");
- ptoken(ts);
-#endif
- return 1;
-}
-
-struct token_state *ts_init(struct token_state *ts, FILE *f) {
- if (!ts) return NULL;
- /* TODO malloc one */
-
- ts->is_string = 0;
- ts->push_back = 0;
- ts->prev_op = 0;
- ts->prev_word = 0;
- ts->quoting = 0;
-
- ts->input = f;
- ts->buf[0] = 0;
- return ts;
-}
-
-int get_token(struct token_state *ts, struct token *tok) {
- int i;
- ts->tok = tok;
- tok->type = 0;
- tok->length = 0;
- ts->prev_op = 0;
- ts->prev_word = 0;
- ts->quoting = 0;
- for (i=0;i<sizeof tok->text; i++) {
- tok->text[i] = 0;
- }
-
- while (!process_char(ts)) {
- }
- return tok->type;
-}
-
-/* true if the character could be used with the given token to make
- * an operator
- */
-static int can_op(int type, int ch) {
- /* TODO */
- switch (type) {
- case TOKEN_Ampersand:
- if (ch == '&') return TOKEN_AND_IF; break;
- case TOKEN_Pipe:
- if (ch == '|') return TOKEN_OR_IF; break;
- case TOKEN_Semicolon:
- if (ch == ';') return TOKEN_DSEMI; break;
- case TOKEN_DLESS:
- if (ch == '-') return TOKEN_DLESSDASH; break;
- case TOKEN_Lessthan:
- switch (ch) {
- case '<': return TOKEN_DLESS; break;
- case '&': return TOKEN_LESSAND; break;
- case '>': return TOKEN_LESSGREAT; break;
- default: break;
- }
- break;
- case TOKEN_Greaterthan:
- switch (ch) {
- case '>': return TOKEN_DGREAT; break;
- case '&': return TOKEN_GREATAND; break;
- case '|': return TOKEN_CLOBBER; break;
- default: break;
- }
- break;
- default:
- break;
- }
- return 0;
-}
-
-static int valid_token(struct token_state *ts) {
- return ts->tok && ts->tok->length;
-}
-
-static int delimit(struct token_state *ts, int ch) {
- if (valid_token(ts)) {
- ts->tok->delimiter = ch;
- if (ts->tok->length == 1 && ts->tok->text[0] == '\n') {
- ts->tok->type = TOKEN_NEWLINE;
- } else if (!ts->tok->nondigits && (ts->tok->delimiter == '>' || ts->tok->delimiter == '<')) {
- ts->tok->type = TOKEN_IO_NUMBER;
- }
-#if DEBUG_TOKEN
- fprintf(stderr, "delimited ");
- ptoken(ts);
-#endif
- return 1;
- }
-
- return 0;
-}
-
-/* return true if delimited */
-static int process_char(struct token_state *ts) {
- int cc;
-
- cc = peek(ts);
-#if 1
- fprintf(stderr, "looking at a '%c'\n", cc);
-#endif
-
-/*
- * If the end of input is recognized, the current token shall be delimited. If
- * there is no current token, the end-of-input indicator shall be returned as
- * the token.
- */
- if (cc == EOF) {
- if (delimit(ts,cc)) {
- return 1;
- }
- fprintf(stderr, "EOF Token\n");
- ts->tok->type = EOF;
- return 1;
- }
-
-/*
- * If the previous character was used as part of an operator and the current
- * character is not quoted and can be used with the current characters to form
- * an operator, it shall be used as part of that (operator) token.
- */
- if (ts->prev_op && !ts->quoting) {
- int newop = can_op(ts->tok->type, cc);
- if (newop) {
- ts->tok->type = newop;
- add_to_token(ts,cc);
- nextch(ts);
- return 0;
- }
- }
-
-/*
- * If the previous character was used as part of an operator and the current
- * character cannot be used with the current characters to form an
- * operator, the operator containing the previous character shall be
- * delimited.
- */
- if (ts->prev_op && ! can_op(ts->tok->type, cc)) {
- if (delimit(ts,cc)) return 1;
- /* TODO error here, should be impossible */
- }
-
- /* match quote */
-/*
- * If the current character is <backslash>, single-quote, or double-quote and
- * it is not quoted, it shall affect quoting for subsequent characters up to
- * the end of the quoted text. The rules for quoting are as described in
- * Quoting . During token recognition no substitutions shall be actually
- * performed, and the result token shall contain exactly the characters that
- * appear in the input (except for <newline> joining), unmodified, including
- * any embedded or enclosing quotes or substitution operators, between the
- * <quotation-mark> and the end of the quoted text. The token shall not be
- * delimited by the end of the quoted field.
- */
- if (!ts->quoting) {
- if (cc == '\\' || cc == '\'' || cc == '"') {
- ts->quoting = cc;
- }
- }
-
- /* match expansion */
-/*
- * If the current character is an unquoted '$' or '`', the shell shall identify
- * the start of any candidates for parameter expansion (Parameter Expansion),
- * command substitution (Command Substitution), or arithmetic expansion
- * (Arithmetic Expansion) from their introductory unquoted character sequences:
- * '$' or "${", "$(" or '`', and "$((", respectively. The shell shall read
- * sufficient input to determine the end of the unit to be expanded (as
- * explained in the cited sections). While processing the characters, if
- * instances of expansions or quoting are found nested within the substitution,
- * the shell shall recursively process them in the manner specified for the
- * construct that is found. The characters found from the beginning of the
- * substitution to its end, allowing for any recursion necessary to recognize
- * embedded constructs, shall be included unmodified in the result token,
- * including any embedded or enclosing substitution operators or quotes. The
- * token shall not be delimited by the end of the substitution.
- */
-#if 0
- if (!ts->quoting) {
- if (cc == '$' || cc == '`') {
- int nc;
- nc = peek(ts);
- switch (nc) {
- case '{':
- ts->expanding = EXP_BRACKET; break;
- case '(':
- ts->expanding = EXP_COMMAND; break;
- default:
- ts->expanding = EXP_WORDVAR; break;
- }
- do_expansion(ts, cc);
- } else if (cc == '`') {
- ts->expanding = EXP_COMMAND;
- }
- }
-#endif
-
-/*
- * If the current character is not quoted and can be used as the first
- * character of a new operator, the current token (if any) shall be delimited.
- * The current character shall be used as the beginning of the next (operator)
- * token.
- */
-
-/*
- * If the current character is an unquoted <newline>, the current token shall
- * be delimited.
- */
-
-/*
- * If the current character is an unquoted <blank>, any token containing the
- * previous character is delimited and the current character shall be
- * discarded.
- */
-
-/*
- * The application shall quote the following characters if they are to
- * represent themselves:
- *
- * | & ; < > ( ) $ ` \ " ' <space> <tab> <newline>
- */
-
-
- if (! ts->quoting) {
- switch (cc) {
- case '&':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- ts->prev_op = 1;
- start_token(ts, TOKEN_Ampersand, cc);
- return 0; break;
- case '|':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- ts->prev_op = 1;
- start_token(ts, TOKEN_Pipe,cc);
- return 0; break;
- case ';':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- ts->prev_op = 1;
- start_token(ts, TOKEN_Semicolon,cc);
- return 0; break;
- case '<':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- ts->prev_op = 1;
- start_token(ts, TOKEN_Lessthan,cc);
- return 0; break;
- case '>':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- ts->prev_op = 1;
- start_token(ts, TOKEN_Greaterthan,cc);
- return 0; break;
- case '\n':
- if (delimit(ts,cc)) return 1;
- fprintf(stderr, "newline\n");
- nextch(ts);
- start_token(ts, TOKEN_NEWLINE,cc);
- if (delimit(ts,cc)) return 1;
- break;
- case '(':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- start_token(ts, TOKEN_Lparen,cc);
- if (delimit(ts,cc)) return 1;
- break;
- case ')':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- start_token(ts, TOKEN_Rparen,cc);
- if (delimit(ts,cc)) return 1;
- break;
- case ' ':
- case '\t':
- if (delimit(ts,cc)) return 1;
- nextch(ts);
- return 0; break;
- default:
- break;
- }
- }
-
-/*
- * If the previous character was part of a word, the current character shall
- * be appended to that word.
- */
- if (ts->prev_word) {
- add_to_token(ts, cc);
- nextch(ts);
- return 0;
- }
-
-/*
- * If the current character is a '#', it and all subsequent characters up
- * to, but excluding, the next <newline> shall be discarded as a comment.
- * The <newline> that ends the line is not considered part of the comment.
- */
- if (cc == '#') {
- cc = nextch(ts);
- cc = peek(ts);
- while (cc != '\n' && cc != EOF) {
- cc = nextch(ts);
- cc = peek(ts);
- }
- return 0;
- }
-
-/*
- * The current character is used as the start of a new word.
- */
- ts->prev_word = 1;
- start_token(ts, TOKEN_WORD, cc);
- nextch(ts);
- return 0;
-}
-
-static int nextchar(struct token_state *ts) {
- if (ts->is_string) {
- if (ts->cursor >= ts->strlen) {
- return EOF;
- }
- return (int)(unsigned char)ts->strinput[ts->cursor++];
- }
-
- return fgetc(ts->input);
-}
-
projects / pdutils / blobdiff