5 * We reserve no LEGAL rights to the Purdue Compiler Construction Tool
6 * Set (PCCTS) -- PCCTS is in the public domain. An individual or
7 * company may do whatever they wish with source code distributed with
8 * PCCTS or the code generated by PCCTS, including the incorporation of
9 * PCCTS, or its output, into commerical software.
11 * We encourage users to develop software with PCCTS. However, we do ask
12 * that credit is given to us for developing PCCTS. By "credit",
13 * we mean that if you incorporate our source code into one of your
14 * programs (commercial product, research project, or otherwise) that you
15 * acknowledge this fact somewhere in the documentation, research report,
16 * etc... If you like PCCTS and have developed a nice tool with the
17 * output, please mention that you developed it using PCCTS. In
18 * addition, we ask that this header remain intact in our source code.
19 * As long as these guidelines are kept, we expect to continue enhancing
20 * this system and expect to make other tools available as they are
25 * Parr Research Corporation
26 * with Purdue University and AHPCRC, University of Minnesota
30 typedef int ANTLRTokenType; // fool AToken.h into compiling
32 #define ANTLR_SUPPORT_CODE
35 #include ATOKENBUFFER_H
36 typedef ANTLRAbstractToken *_ANTLRTokenPtr;
38 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
39 static unsigned char test[1000];
42 #ifdef DBG_REFCOUNTTOKEN
43 int ANTLRCommonToken::ctor = 0;
44 int ANTLRCommonToken::dtor = 0;
48 ANTLRTokenBuffer(ANTLRTokenStream *_input, int _k, int _cs)
52 buffer_size = chunk_size = _cs;
53 buffer = (_ANTLRTokenPtr *)
54 calloc(chunk_size+1,sizeof(_ANTLRTokenPtr ));
55 if ( buffer == NULL ) {
56 panic("cannot alloc token buffer");
58 buffer++; // leave the first elem empty so tp-1 is valid ptr
64 end_of_buffer = &buffer[buffer_size-1];
65 threshold = &buffer[(int)(buffer_size*(1.0/2.0))];
66 _deleteTokens = 1; // assume we delete tokens
74 // Delete all remaining tokens (from 0..last inclusive)
78 for (z=buffer; z<=last; z++)
82 #ifdef DBG_REFCOUNTTOKEN
83 fprintf(stderr, "##########dtor: deleting token '%s' (ref %d)\n",
84 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
86 if ( (*z)->nref()==0 )
93 if ( buffer!=NULL ) free((char *)(buffer-1));
96 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
100 _ANTLRTokenPtr ANTLRTokenBuffer::
103 if ( tp <= last ) // is there any buffered lookahead still to be read?
105 return *tp++; // read buffered lookahead
107 // out of buffered lookahead, get some more "real"
108 // input from getANTLRToken()
109 if ( num_markers==0 )
111 if( next > threshold )
114 fprintf(stderr,"getToken: next > threshold (high water is %d)\n", threshold-buffer);
120 if ( next > end_of_buffer )
123 fprintf(stderr,"getToken: next > end_of_buffer (size is %d)\n", buffer_size);
128 *next = getANTLRToken();
129 (*next)->ref(); // say we have a copy of this pointer in buffer
136 void ANTLRTokenBuffer::
139 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
140 fprintf(stderr, "rewind(%d)[nm=%d,from=%d,%d.n=%d]\n", pos, num_markers, tp-buffer,pos,test[pos]);
148 * This function is used to specify that the token pointers read
149 * by the ANTLRTokenBuffer should be buffered up (to be reused later).
151 int ANTLRTokenBuffer::
154 #if defined(DBG_TBUF)||defined(DBG_TBUF_MARK_REW)
156 fprintf(stderr,"mark(%d)[nm=%d,%d.n=%d]\n",tp-buffer,num_markers+1,tp-buffer,test[tp-buffer]);
163 * returns the token pointer n positions ahead.
164 * This implies that bufferedToken(1) gets the NEXT symbol of lookahead.
165 * This is used in conjunction with the ANTLRParser lookahead buffer.
167 * No markers are set or anything. A bunch of input is buffered--that's all.
168 * The tp pointer is left alone as the lookahead has not been advanced
169 * with getToken(). The next call to getToken() will find a token
170 * in the buffer and won't have to call getANTLRToken().
172 * If this is called before a consume() is done, how_many_more_i_need is
175 _ANTLRTokenPtr ANTLRTokenBuffer::
178 // int how_many_more_i_need = (last-tp < 0) ? n : n-(last-tp)-1;
179 int how_many_more_i_need = (tp > last) ? n : n-(last-tp)-1;
180 // Make sure that at least n tokens are available in the buffer
182 fprintf(stderr, "bufferedToken(%d)\n", n);
184 for (int i=1; i<=how_many_more_i_need; i++)
186 if ( next > end_of_buffer ) // buffer overflow?
190 *next = getANTLRToken();
191 (*next)->ref(); // say we have a copy of this pointer in buffer
198 /* If no markers are set, the none of the input needs to be saved (except
199 * for the lookahead Token pointers). We save only k-1 token pointers as
200 * we are guaranteed to do a getANTLRToken() right after this because otherwise
201 * we wouldn't have needed to extend the buffer.
203 * If there are markers in the buffer, we need to save things and so
204 * extendBuffer() is called.
206 void ANTLRTokenBuffer::
210 fprintf(stderr, "in makeRoom.................\n");
211 fprintf(stderr, "num_markers==%d\n", num_markers);
214 if ( num_markers == 0 )
218 fprintf(stderr, "moving lookahead and resetting next\n");
221 fprintf(stderr, "tbuf = [");
222 for (r=buffer; r<=last; r++)
224 if ( *r==NULL ) fprintf(stderr, " xxx");
225 else fprintf(stderr, " '%s'", ((ANTLRCommonToken *)*r)->getText());
227 fprintf(stderr, " ]\n");
230 "before: tp=%d, last=%d, next=%d, threshold=%d\n",tp-buffer,last-buffer,next-buffer,threshold-buffer);
233 // Delete all tokens from 0..last-(k-1) inclusive
237 for (z=buffer; z<=last-(k-1); z++)
241 #ifdef DBG_REFCOUNTTOKEN
242 fprintf(stderr, "##########makeRoom: deleting token '%s' (ref %d)\n",
243 ((ANTLRCommonToken *)*z)->getText(), (*z)->nref());
245 if ( (*z)->nref()==0 )
252 // reset the buffer to initial conditions, but move k-1 symbols
253 // to the beginning of buffer and put new input symbol at k
254 _ANTLRTokenPtr *p = buffer, *q = last-(k-1)+1;
255 // ANTLRAbstractToken **p = buffer, **q = end_of_buffer-(k-1)+1;
257 fprintf(stderr, "lookahead buffer = [");
259 for (int i=1; i<=(k-1); i++)
264 " '%s'", ((ANTLRCommonToken *)buffer[i-1])->getText());
268 fprintf(stderr, " ]\n");
271 tp = &buffer[k-1]; // tp points to what will be filled in next
275 "after: tp=%d, last=%d, next=%d\n",
276 tp-buffer, last-buffer, next-buffer);
286 /* This function extends 'buffer' by chunk_size and returns with all
287 * pointers at the same relative positions in the buffer (the buffer base
288 * address could have changed in realloc()) except that 'next' comes
289 * back set to where the next token should be stored. All other pointers
296 int save_last = last-buffer, save_tp = tp-buffer, save_next = next-buffer;
298 fprintf(stderr, "extending physical buffer\n");
300 buffer_size += chunk_size;
301 buffer = (_ANTLRTokenPtr *)
302 realloc((char *)(buffer-1),
303 (buffer_size+1)*sizeof(_ANTLRTokenPtr ));
304 if ( buffer == NULL ) {
305 panic("cannot alloc token buffer");
307 buffer++; // leave the first elem empty so tp-1 is valid ptr
309 tp = buffer + save_tp; // put the pointers back to same relative position
310 last = buffer + save_last;
311 next = buffer + save_next;
312 end_of_buffer = &buffer[buffer_size-1];
313 threshold = &buffer[(int)(buffer_size*(1.0/2.0))];
316 // zero out new token ptrs so we'll know if something to delete in buffer
317 ANTLRAbstractToken **p = end_of_buffer-chunk_size+1;
318 for (; p<=end_of_buffer; p++) *p = NULL;
322 /* to avoid having to link in another file just for the smart token ptr
323 * stuff, we include it here. Ugh.