diff options
| -rw-r--r-- | TODO | 6 | ||||
| -rw-r--r-- | doc/flex.texi | 94 | ||||
| -rw-r--r-- | flex.skl | 269 | ||||
| -rw-r--r-- | gen.c | 8 | ||||
| -rw-r--r-- | main.c | 6 |
5 files changed, 296 insertions, 87 deletions
@@ -52,15 +52,13 @@ ** make test suite more complete +** add a test for yypush_buffer_state/yypop_buffer_state + * generic coding ** move as much skeleton code as possible out of gen.c and into flex.skl -** Automatic management of the input buffer stack. (currently does - this for the start condition stack, but not for the input - buffer stack.) - ** figure out whether we want to add the capability to have auto-generated backout rules diff --git a/doc/flex.texi b/doc/flex.texi index 9a0d057..cd3ecad 100644 --- a/doc/flex.texi +++ b/doc/flex.texi @@ -1928,12 +1928,14 @@ using: @deftypefun void yy_switch_to_buffer ( YY_BUFFER_STATE new_buffer ) @end deftypefun -The above switches the scanner's input buffer so subsequent tokens will -come from @code{new_buffer}. Note that @code{yy_switch_to_buffer()} may -be used by @code{yywrap()} to set things up for continued scanning, -instead of opening a new file and pointing @file{yyin} at it. Note also -that switching input sources via either @code{yy_switch_to_buffer()} or -@code{yywrap()} does @emph{not} change the start condition. +The above function switches the scanner's input buffer so subsequent tokens +will come from @code{new_buffer}. Note that @code{yy_switch_to_buffer()} may +be used by @code{yywrap()} to set things up for continued scanning, instead of +opening a new file and pointing @file{yyin} at it. If you are looking for a +stack of input buffers, then you want to use @code{yypush_buffer_state()} +instead of this function. Note also that switching input sources via either +@code{yy_switch_to_buffer()} or @code{yywrap()} does @emph{not} change the +start condition. @cindex memory, deleting input buffers @deftypefun void yy_delete_buffer ( YY_BUFFER_STATE buffer ) @@ -1943,6 +1945,26 @@ is used to reclaim the storage associated with a buffer. (@code{buffer} can be NULL, in which case the routine does nothing.) You can also clear the current contents of a buffer using: +@cindex pushing an input buffer +@cindex stack, input buffer push +@deftypefun void yypush_buffer_state ( YY_BUFFER_STATE buffer ) +@end deftypefun + +This function pushes the new buffer state onto an internal stack. The pushed +state becomes the new current state. The stack is maintained by flex and will +grow as required. This function is intended to be used instead of +@code{yy_switch_to_buffer}, when you want to change states, but preserve the +current state for later use. + +@cindex popping an input buffer +@cindex stack, input buffer pop +@deftypefun void yypop_buffer_state ( ) +@end deftypefun + +This function removes the current state from the top of the stack, and deletes +it by calling @code{yy_delete_buffer}. The next state on the stack, if any, +becomes the new current state. + @cindex clearing an input buffer @cindex flushing an input buffer @deftypefun void yy_flush_buffer ( YY_BUFFER_STATE buffer ) @@ -1960,15 +1982,57 @@ is an alias for @code{yy_create_buffer()}, provided for compatibility with the C++ use of @code{new} and @code{delete} for creating and destroying dynamic objects. -@cindex YY_CURRENT_BUFFER, and multiple buffers -Finally, the macro @code{YY_CURRENT_BUFFER} macro returns a -@code{YY_BUFFER_STATE} handle to the current buffer. +@cindex YY_CURRENT_BUFFER, and multiple buffers Finally, the macro +@code{YY_CURRENT_BUFFER} macro returns a @code{YY_BUFFER_STATE} handle to the +current buffer. It should not be used as an lvalue. @cindex EOF, example using multiple input buffers -Here is an example of using these features for writing a scanner +Here are two examples of using these features for writing a scanner which expands include files (the @code{<<EOF>>} -feature is discussed below): +feature is discussed below). + +This first example uses yypush_buffer_state and yypop_buffer_state. Flex +maintains the stack internally. + +@cindex handling include files with multiple input buffers +@example +@verbatim + /* the "incl" state is used for picking up the name + * of an include file + */ + %x incl + %% + include BEGIN(incl); + + [a-z]+ ECHO; + [^a-z\n]*\n? ECHO; + + <incl>[ \t]* /* eat the whitespace */ + <incl>[^ \t\n]+ { /* got the include file name */ + yyin = fopen( yytext, "r" ); + + if ( ! yyin ) + error( ... ); + + yypush_buffer_state(yy_create_buffer( yyin, YY_BUF_SIZE )); + + BEGIN(INITIAL); + } + + <<EOF>> { + yypop_buffer_state(); + + if ( !YY_CURRENT_BUFFER ) + { + yyterminate(); + } + } +@end verbatim +@end example + +The second example, below, does the same thing as the previous example did, but +manages its own input buffer stack manually (instead of letting flex do it). @cindex handling include files with multiple input buffers @example @@ -5882,7 +5946,7 @@ you might try this: @example @verbatim /* For non-reentrant C scanner only. */ -yy_delete_buffer(yy_current_buffer); +yy_delete_buffer(YY_CURRENT_BUFFER); yy_init = 1; @end verbatim @end example @@ -5898,7 +5962,7 @@ situation. It is possible that some other globals may need resetting as well. > We thought that it would be possible to have this number through the > evaluation of the following expression: > -> seek_position = (no_buffers)*YY_READ_BUF_SIZE + yy_c_buf_p - yy_current_buffer->yy_ch_buf +> seek_position = (no_buffers)*YY_READ_BUF_SIZE + yy_c_buf_p - YY_CURRENT_BUFFER->yy_ch_buf @end verbatim @end example @@ -5909,7 +5973,7 @@ even though @code{YY_READ_BUF_SIZE} bytes were requested). The second problem is that when refilling its internal buffer, @code{flex} keeps some characters from the previous buffer (because usually it's in the middle of a match, and needs those characters to construct @code{yytext} for the match once it's -done). Because of this, @code{yy_c_buf_p - yy_current_buffer->yy_ch_buf} won't +done). Because of this, @code{yy_c_buf_p - YY_CURRENT_BUFFER->yy_ch_buf} won't be exactly the number of characters already read from the current buffer. An alternative solution is to count the number of characters you've matched @@ -6267,7 +6331,7 @@ Date: Wed, 13 Nov 1996 19:51:54 PST From: Vern Paxson <vern> > "unput()" them to input flow, question occurs. If I do this after I scan -> a carriage, the variable "yy_current_buffer->yy_at_bol" is changed. That +> a carriage, the variable "YY_CURRENT_BUFFER->yy_at_bol" is changed. That > means the carriage flag has gone. You can control this by calling yy_set_bol(). It's described in the manual. @@ -393,16 +393,28 @@ struct yy_buffer_state %c-only Standard (non-C++) definition %not-for-header %if-not-reentrant -static YY_BUFFER_STATE yy_current_buffer = 0; + +/* Stack of input buffers. */ +static size_t yy_buffer_stack_top = 0; /*<< index of top of stack. */ +static size_t yy_buffer_stack_max = 0; /*<< capacity of stack. */ +static YY_BUFFER_STATE * yy_buffer_stack = 0; /*<< Stack as an array. */ %endif %ok-for-header %pop - /* We provide macros for accessing buffer states in case in the * future we want to put the buffer states in a more general * "scanner state". + * + * Returns the top of the stack, or NULL. + */ +#define YY_CURRENT_BUFFER ( YY_G(yy_buffer_stack) \ + ? YY_G(yy_buffer_stack)[YY_G(yy_buffer_stack_top)] \ + : NULL) + +/* Same as previous macro, but useful when we know that the buffer stack is not + * NULL or when we need an lvalue. For internal use only. */ -#define YY_CURRENT_BUFFER yy_current_buffer +#define YY_CURRENT_BUFFER_FAST YY_G(yy_buffer_stack)[YY_G(yy_buffer_stack_top)] %push %c-only Standard (non-C++) definition @@ -438,8 +450,11 @@ YY_BUFFER_STATE yy_create_buffer YY_PARAMS(( FILE *file, int size YY_PROTO_LAST_ void yy_delete_buffer YY_PARAMS(( YY_BUFFER_STATE b YY_PROTO_LAST_ARG )); void yy_init_buffer YY_PARAMS(( YY_BUFFER_STATE b, FILE *file YY_PROTO_LAST_ARG )); void yy_flush_buffer YY_PARAMS(( YY_BUFFER_STATE b YY_PROTO_LAST_ARG )); +void yypush_buffer_state YY_PARAMS(( YY_BUFFER_STATE new_buffer YY_PROTO_LAST_ARG )); +void yypop_buffer_state YY_PARAMS(( YY_PROTO_ONLY_ARG )); +void yyensure_buffer_stack YY_PARAMS(( YY_PROTO_ONLY_ARG )); -#define YY_FLUSH_BUFFER yy_flush_buffer( YY_G(yy_current_buffer) YY_CALL_LAST_ARG) +#define YY_FLUSH_BUFFER yy_flush_buffer( YY_CURRENT_BUFFER YY_CALL_LAST_ARG) YY_BUFFER_STATE yy_scan_buffer YY_PARAMS(( char *base, yy_size_t size YY_PROTO_LAST_ARG )); YY_BUFFER_STATE yy_scan_string YY_PARAMS(( yyconst char *yy_str YY_PROTO_LAST_ARG )); @@ -455,21 +470,25 @@ void yyfree YY_PARAMS(( void * YY_PROTO_LAST_ARG )); #define yy_set_interactive(is_interactive) \ { \ - if ( ! YY_G(yy_current_buffer) ) \ - YY_G(yy_current_buffer) = \ + if ( ! YY_CURRENT_BUFFER ){ \ + yyensure_buffer_stack (YY_CALL_ONLY_ARG); \ + YY_CURRENT_BUFFER_FAST = \ yy_create_buffer( yyin, YY_BUF_SIZE YY_CALL_LAST_ARG); \ - YY_G(yy_current_buffer)->yy_is_interactive = is_interactive; \ + } \ + YY_CURRENT_BUFFER_FAST->yy_is_interactive = is_interactive; \ } #define yy_set_bol(at_bol) \ { \ - if ( ! YY_G(yy_current_buffer) ) \ - YY_G(yy_current_buffer) = \ + if ( ! YY_CURRENT_BUFFER ){\ + yyensure_buffer_stack (YY_CALL_ONLY_ARG); \ + YY_CURRENT_BUFFER_FAST = \ yy_create_buffer( yyin, YY_BUF_SIZE YY_CALL_LAST_ARG); \ - YY_G(yy_current_buffer)->yy_at_bol = at_bol; \ + } \ + YY_CURRENT_BUFFER_FAST->yy_at_bol = at_bol; \ } -#define YY_AT_BOL() (YY_G(yy_current_buffer)->yy_at_bol) +#define YY_AT_BOL() (YY_CURRENT_BUFFER_FAST->yy_at_bol) %% [1.0] yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here @@ -531,7 +550,9 @@ struct yyguts_t /* The rest are the same as the globals declared in the non-reentrant scanner. */ FILE *yyin_r, *yyout_r; - YY_BUFFER_STATE yy_current_buffer; + size_t yy_buffer_stack_top; /*<< index of top of stack. */ + size_t yy_buffer_stack_max; /*<< capacity of stack. */ + YY_BUFFER_STATE * yy_buffer_stack; /*<< Stack as an array. */ char yy_hold_char; int yy_n_chars; int yyleng_r; @@ -934,9 +955,11 @@ YY_DECL yyout = & std::cout; %pop - if ( ! YY_G(yy_current_buffer) ) - YY_G(yy_current_buffer) = + if ( ! YY_CURRENT_BUFFER ) { + yyensure_buffer_stack (YY_CALL_ONLY_ARG); + YY_CURRENT_BUFFER_FAST = yy_create_buffer( yyin, YY_BUF_SIZE YY_CALL_LAST_ARG); + } yy_load_buffer_state( YY_CALL_ONLY_ARG ); } @@ -980,20 +1003,20 @@ do_action: /* This label is used only to access EOF actions. */ *yy_cp = YY_G(yy_hold_char); YY_RESTORE_YY_MORE_OFFSET - if ( YY_G(yy_current_buffer)->yy_buffer_status == YY_BUFFER_NEW ) + if ( YY_CURRENT_BUFFER_FAST->yy_buffer_status == YY_BUFFER_NEW ) { /* We're scanning a new file or input source. It's * possible that this happened because the user * just pointed yyin at a new source and called * yylex(). If so, then we have to assure - * consistency between yy_current_buffer and our + * consistency between YY_CURRENT_BUFFER and our * globals. Here is the right place to do so, because * this is the first action (other than possibly a * back-up) that will match for the new input source. */ - YY_G(yy_n_chars) = YY_G(yy_current_buffer)->yy_n_chars; - YY_G(yy_current_buffer)->yy_input_file = yyin; - YY_G(yy_current_buffer)->yy_buffer_status = YY_BUFFER_NORMAL; + YY_G(yy_n_chars) = YY_CURRENT_BUFFER_FAST->yy_n_chars; + YY_CURRENT_BUFFER_FAST->yy_input_file = yyin; + YY_CURRENT_BUFFER_FAST->yy_buffer_status = YY_BUFFER_NORMAL; } /* Note that here we test for yy_c_buf_p "<=" to the position @@ -1003,7 +1026,7 @@ do_action: /* This label is used only to access EOF actions. */ * end-of-buffer state). Contrast this with the test * in input(). */ - if ( YY_G(yy_c_buf_p) <= &YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars)] ) + if ( YY_G(yy_c_buf_p) <= &YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars)] ) { /* This was really a NUL. */ yy_state_type yy_next_state; @@ -1082,7 +1105,7 @@ do_action: /* This label is used only to access EOF actions. */ case EOB_ACT_LAST_MATCH: YY_G(yy_c_buf_p) = - &YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars)]; + &YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars)]; yy_current_state = yy_get_previous_state( YY_CALL_ONLY_ARG ); @@ -1205,16 +1228,16 @@ static int yy_get_next_buffer YYFARGS0(void) int yyFlexLexer::yy_get_next_buffer() %pop { - register char *dest = YY_G(yy_current_buffer)->yy_ch_buf; + register char *dest = YY_CURRENT_BUFFER_FAST->yy_ch_buf; register char *source = YY_G(yytext_ptr); register int number_to_move, i; int ret_val; - if ( YY_G(yy_c_buf_p) > &YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars) + 1] ) + if ( YY_G(yy_c_buf_p) > &YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars) + 1] ) YY_FATAL_ERROR( "fatal flex scanner internal error--end of buffer missed" ); - if ( YY_G(yy_current_buffer)->yy_fill_buffer == 0 ) + if ( YY_CURRENT_BUFFER_FAST->yy_fill_buffer == 0 ) { /* Don't try to fill the buffer, so this is an EOF. */ if ( YY_G(yy_c_buf_p) - YY_G(yytext_ptr) - YY_MORE_ADJ == 1 ) { @@ -1241,16 +1264,16 @@ int yyFlexLexer::yy_get_next_buffer() for ( i = 0; i < number_to_move; ++i ) *(dest++) = *(source++); - if ( YY_G(yy_current_buffer)->yy_buffer_status == YY_BUFFER_EOF_PENDING ) + if ( YY_CURRENT_BUFFER_FAST->yy_buffer_status == YY_BUFFER_EOF_PENDING ) /* don't do the read, it's not guaranteed to return an EOF, * just force an EOF */ - YY_G(yy_current_buffer)->yy_n_chars = YY_G(yy_n_chars) = 0; + YY_CURRENT_BUFFER_FAST->yy_n_chars = YY_G(yy_n_chars) = 0; else { size_t num_to_read = - YY_G(yy_current_buffer)->yy_buf_size - number_to_move - 1; + YY_CURRENT_BUFFER_FAST->yy_buf_size - number_to_move - 1; while ( num_to_read <= 0 ) { /* Not enough room in the buffer - grow it. */ @@ -1260,7 +1283,7 @@ int yyFlexLexer::yy_get_next_buffer() #else /* just a shorter name for the current buffer */ - YY_BUFFER_STATE b = YY_G(yy_current_buffer); + YY_BUFFER_STATE b = YY_CURRENT_BUFFER; int yy_c_buf_p_offset = (int) (YY_G(yy_c_buf_p) - b->yy_ch_buf); @@ -1289,7 +1312,7 @@ int yyFlexLexer::yy_get_next_buffer() YY_G(yy_c_buf_p) = &b->yy_ch_buf[yy_c_buf_p_offset]; - num_to_read = YY_G(yy_current_buffer)->yy_buf_size - + num_to_read = YY_CURRENT_BUFFER_FAST->yy_buf_size - number_to_move - 1; #endif } @@ -1298,10 +1321,10 @@ int yyFlexLexer::yy_get_next_buffer() num_to_read = YY_READ_BUF_SIZE; /* Read in more data. */ - YY_INPUT( (&YY_G(yy_current_buffer)->yy_ch_buf[number_to_move]), + YY_INPUT( (&YY_CURRENT_BUFFER_FAST->yy_ch_buf[number_to_move]), YY_G(yy_n_chars), num_to_read ); - YY_G(yy_current_buffer)->yy_n_chars = YY_G(yy_n_chars); + YY_CURRENT_BUFFER_FAST->yy_n_chars = YY_G(yy_n_chars); } if ( YY_G(yy_n_chars) == 0 ) @@ -1315,7 +1338,7 @@ int yyFlexLexer::yy_get_next_buffer() else { ret_val = EOB_ACT_LAST_MATCH; - YY_G(yy_current_buffer)->yy_buffer_status = + YY_CURRENT_BUFFER_FAST->yy_buffer_status = YY_BUFFER_EOF_PENDING; } } @@ -1324,10 +1347,10 @@ int yyFlexLexer::yy_get_next_buffer() ret_val = EOB_ACT_CONTINUE_SCAN; YY_G(yy_n_chars) += number_to_move; - YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; - YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars) + 1] = YY_END_OF_BUFFER_CHAR; - YY_G(yytext_ptr) = &YY_G(yy_current_buffer)->yy_ch_buf[0]; + YY_G(yytext_ptr) = &YY_CURRENT_BUFFER_FAST->yy_ch_buf[0]; return ret_val; } @@ -1389,24 +1412,24 @@ int yyFlexLexer::yy_get_next_buffer() /* undo effects of setting up yytext */ *yy_cp = YY_G(yy_hold_char); - if ( yy_cp < YY_G(yy_current_buffer)->yy_ch_buf + 2 ) + if ( yy_cp < YY_CURRENT_BUFFER_FAST->yy_ch_buf + 2 ) { /* need to shift things up to make room */ /* +2 for EOB chars. */ register int number_to_move = YY_G(yy_n_chars) + 2; - register char *dest = &YY_G(yy_current_buffer)->yy_ch_buf[ - YY_G(yy_current_buffer)->yy_buf_size + 2]; + register char *dest = &YY_CURRENT_BUFFER_FAST->yy_ch_buf[ + YY_CURRENT_BUFFER_FAST->yy_buf_size + 2]; register char *source = - &YY_G(yy_current_buffer)->yy_ch_buf[number_to_move]; + &YY_CURRENT_BUFFER_FAST->yy_ch_buf[number_to_move]; - while ( source > YY_G(yy_current_buffer)->yy_ch_buf ) + while ( source > YY_CURRENT_BUFFER_FAST->yy_ch_buf ) *--dest = *--source; yy_cp += (int) (dest - source); yy_bp += (int) (dest - source); - YY_G(yy_current_buffer)->yy_n_chars = - YY_G(yy_n_chars) = YY_G(yy_current_buffer)->yy_buf_size; + YY_CURRENT_BUFFER_FAST->yy_n_chars = + YY_G(yy_n_chars) = YY_CURRENT_BUFFER_FAST->yy_buf_size; - if ( yy_cp < YY_G(yy_current_buffer)->yy_ch_buf + 2 ) + if ( yy_cp < YY_CURRENT_BUFFER_FAST->yy_ch_buf + 2 ) YY_FATAL_ERROR( "flex scanner push-back overflow" ); } @@ -1450,7 +1473,7 @@ int yyFlexLexer::yy_get_next_buffer() * If this occurs *before* the EOB characters, then it's a * valid NUL; if not, then we've hit the end of the buffer. */ - if ( YY_G(yy_c_buf_p) < &YY_G(yy_current_buffer)->yy_ch_buf[YY_G(yy_n_chars)] ) + if ( YY_G(yy_c_buf_p) < &YY_CURRENT_BUFFER_FAST->yy_ch_buf[YY_G(yy_n_chars)] ) /* This was really a NUL. */ *YY_G(yy_c_buf_p) = '\0'; @@ -1518,11 +1541,13 @@ int yyFlexLexer::yy_get_next_buffer() void yyFlexLexer::yyrestart( std::istream* input_file ) %pop { - if ( ! YY_G(yy_current_buffer) ) - YY_G(yy_current_buffer) = + if ( ! YY_CURRENT_BUFFER ){ + yyensure_buffer_stack (YY_CALL_ONLY_ARG); + YY_CURRENT_BUFFER_FAST = yy_create_buffer( yyin, YY_BUF_SIZE YY_CALL_LAST_ARG); + } - yy_init_buffer( YY_G(yy_current_buffer), input_file YY_CALL_LAST_ARG); + yy_init_buffer( YY_CURRENT_BUFFER, input_file YY_CALL_LAST_ARG); yy_load_buffer_state( YY_CALL_ONLY_ARG ); } @@ -1533,18 +1558,24 @@ int yyFlexLexer::yy_get_next_buffer() void yyFlexLexer::yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) %pop { - if ( YY_G(yy_current_buffer) == new_buffer ) + /* TODO. We should be able to replace this entire function body + * with + * yypop_buffer_state(); + * yypush_buffer_state(new_buffer); + */ + yyensure_buffer_stack (YY_CALL_ONLY_ARG); + if ( YY_CURRENT_BUFFER == new_buffer ) return; - if ( YY_G(yy_current_buffer) ) + if ( YY_CURRENT_BUFFER ) { /* Flush out information for old buffer. */ *YY_G(yy_c_buf_p) = YY_G(yy_hold_char); - YY_G(yy_current_buffer)->yy_buf_pos = YY_G(yy_c_buf_p); - YY_G(yy_current_buffer)->yy_n_chars = YY_G(yy_n_chars); + YY_CURRENT_BUFFER_FAST->yy_buf_pos = YY_G(yy_c_buf_p); + YY_CURRENT_BUFFER_FAST->yy_n_chars = YY_G(yy_n_chars); } - YY_G(yy_current_buffer) = new_buffer; + YY_CURRENT_BUFFER_FAST = new_buffer; yy_load_buffer_state( YY_CALL_ONLY_ARG ); /* We don't actually know whether we did this switch during @@ -1563,9 +1594,9 @@ int yyFlexLexer::yy_get_next_buffer() void yyFlexLexer::yy_load_buffer_state() %pop { - YY_G(yy_n_chars) = YY_G(yy_current_buffer)->yy_n_chars; - YY_G(yytext_ptr) = YY_G(yy_c_buf_p) = YY_G(yy_current_buffer)->yy_buf_pos; - yyin = YY_G(yy_current_buffer)->yy_input_file; + YY_G(yy_n_chars) = YY_CURRENT_BUFFER_FAST->yy_n_chars; + YY_G(yytext_ptr) = YY_G(yy_c_buf_p) = YY_CURRENT_BUFFER_FAST->yy_buf_pos; + yyin = YY_CURRENT_BUFFER_FAST->yy_input_file; YY_G(yy_hold_char) = *YY_G(yy_c_buf_p); } @@ -1608,8 +1639,8 @@ int yyFlexLexer::yy_get_next_buffer() if ( ! b ) return; - if ( b == YY_G(yy_current_buffer) ) - YY_G(yy_current_buffer) = (YY_BUFFER_STATE) 0; + if ( b == YY_CURRENT_BUFFER ) /* Not sure if we should pop here. */ + YY_CURRENT_BUFFER_FAST = (YY_BUFFER_STATE) 0; if ( b->yy_is_our_buffer ) yyfree( (void *) b->yy_ch_buf YY_CALL_LAST_ARG ); @@ -1688,10 +1719,111 @@ void yyFlexLexer::yy_init_buffer( YY_BUFFER_STATE b, std::istream* file ) b->yy_at_bol = 1; b->yy_buffer_status = YY_BUFFER_NEW; - if ( b == YY_G(yy_current_buffer) ) + if ( b == YY_CURRENT_BUFFER ) yy_load_buffer_state( YY_CALL_ONLY_ARG ); } +%push +%c-only +/** Pushes the new state onto the stack. The new state becomes + * the current state. This function will allocate the stack + * if necessary. + */ +void yypush_buffer_state YYFARGS1(YY_BUFFER_STATE,new_buffer) +{ + if (new_buffer == NULL) + return; + + yyensure_buffer_stack(YY_CALL_ONLY_ARG); + + /* This block is copied from yy_switch_to_buffer. */ + if ( YY_CURRENT_BUFFER ) + { + /* Flush out information for old buffer. */ + *YY_G(yy_c_buf_p) = YY_G(yy_hold_char); + YY_CURRENT_BUFFER_FAST->yy_buf_pos = YY_G(yy_c_buf_p); + YY_CURRENT_BUFFER_FAST->yy_n_chars = YY_G(yy_n_chars); + } + + /* Only push if top exists. Otherwise, replace top. */ + if (YY_CURRENT_BUFFER) + YY_G(yy_buffer_stack_top)++; + YY_CURRENT_BUFFER_FAST = new_buffer; + + /* copied from yy_switch_to_buffer. */ + yy_load_buffer_state( YY_CALL_ONLY_ARG ); + YY_G(yy_did_buffer_switch_on_eof) = 1; +} +%pop + +%push +%c-only +/** Removes and DELETES the top of the stack, if present. + * The next element becomes the new top, if present. + */ +void yypop_buffer_state YYFARGS0(void) +{ + if (!YY_CURRENT_BUFFER) + return; + + yy_delete_buffer(YY_CURRENT_BUFFER YY_CALL_LAST_ARG); + YY_CURRENT_BUFFER_FAST = NULL; + if (YY_G(yy_buffer_stack_top) > 0) + --YY_G(yy_buffer_stack_top); + + if (YY_CURRENT_BUFFER) { + yy_load_buffer_state( YY_CALL_ONLY_ARG ); + YY_G(yy_did_buffer_switch_on_eof) = 1; + } +} +%pop + +%push +%c-only +/** Allocates the stack if it does not exist. + * Guarantees space for at least one push. + */ +void yyensure_buffer_stack YYFARGS0(void) +{ + int num_to_alloc; + + if (!YY_G(yy_buffer_stack)) { + + /* First allocation is just for 2 elements, since we don't know if this + * scanner will even need a stack. We use 2 instead of 1 to avoid an + * immediate realloc on the next call. + */ + num_to_alloc = 1; + YY_G(yy_buffer_stack) = (struct yy_buffer_state**)yyalloc + (num_to_alloc * sizeof(struct yy_buffer_state*) + YY_CALL_LAST_ARG); + + memset(YY_G(yy_buffer_stack), 0, num_to_alloc * sizeof(struct yy_buffer_state*)); + + YY_G(yy_buffer_stack_max) = num_to_alloc; + YY_G(yy_buffer_stack_top) = 0; + return; + } + + if (YY_G(yy_buffer_stack_top) >= (YY_G(yy_buffer_stack_max)) - 1){ + + /* Increase the buffer to prepare for a possible push. */ + int grow_size = 8 /* arbitrary grow size */; + + num_to_alloc = YY_G(yy_buffer_stack_max) + grow_size; + YY_G(yy_buffer_stack) = (struct yy_buffer_state**)yyrealloc + (YY_G(yy_buffer_stack), + num_to_alloc * sizeof(struct yy_buffer_state*) + YY_CALL_LAST_ARG); + + /* zero only the new slots.*/ + memset(YY_G(yy_buffer_stack) + YY_G(yy_buffer_stack_max), 0, grow_size * sizeof(struct yy_buffer_state*)); + YY_G(yy_buffer_stack_max) = num_to_alloc; + } +} +%pop + + #ifndef YY_NO_SCAN_BUFFER %push @@ -2020,7 +2152,9 @@ static int yy_init_globals YYFARGS0(void) #ifdef YY_USE_LINENO yylineno = 1; #endif - YY_G(yy_current_buffer) = 0; + YY_G(yy_buffer_stack) = 0; + YY_G(yy_buffer_stack_top) = 0; + YY_G(yy_buffer_stack_max) = 0; YY_G(yy_c_buf_p) = (char *) 0; YY_G(yy_init) = 1; YY_G(yy_start) = 0; @@ -2093,16 +2227,23 @@ int yylex_init( ptr_yy_globals ) /* yylex_destroy is for both reentrant and non-reentrant scanners. */ int yylex_destroy YYFARGS0(void) { - /* Destroy the current (main) buffer. */ - yy_delete_buffer( YY_G(yy_current_buffer) YY_CALL_LAST_ARG ); - YY_G(yy_current_buffer) = NULL; + int i; + + /* Pop the buffer stack, destroying each element. */ + while(YY_CURRENT_BUFFER){ + yy_delete_buffer( YY_CURRENT_BUFFER YY_CALL_LAST_ARG ); + YY_CURRENT_BUFFER_FAST = NULL; + yypop_buffer_state(YY_CALL_ONLY_ARG); + } + + /* Destroy the stack itself. */ + yyfree(YY_G(yy_buffer_stack) YY_CALL_LAST_ARG); + YY_G(yy_buffer_stack) = NULL; #if defined(YY_STACK_USED) || defined(YY_REENTRANT) /* Destroy the start condition stack. */ - if (YY_G(yy_start_stack) ){ yyfree( YY_G(yy_start_stack) YY_CALL_LAST_ARG ); YY_G(yy_start_stack) = NULL; - } #endif #ifdef YY_USES_REJECT @@ -1898,7 +1898,7 @@ void make_tables () } else { - outn ("\tif ( YY_G(yy_current_buffer)->yy_is_interactive ) \\"); + outn ("\tif ( YY_CURRENT_BUFFER_LVALUE->yy_is_interactive ) \\"); outn ("\t\t{ \\"); outn ("\t\tint c = '*'; \\"); outn ("\t\tsize_t n; \\"); @@ -1935,7 +1935,7 @@ void make_tables () if (bol_needed) { indent_puts ("if ( yyleng > 0 ) \\"); indent_up (); - indent_puts ("YY_G(yy_current_buffer)->yy_at_bol = \\"); + indent_puts ("YY_CURRENT_BUFFER_LVALUE->yy_at_bol = \\"); indent_puts ("\t\t(yytext[yyleng - 1] == '\\n'); \\"); indent_down (); } @@ -2149,10 +2149,10 @@ void make_tables () /* Update BOL and yylineno inside of input(). */ if (bol_needed) { indent_puts - ("YY_G(yy_current_buffer)->yy_at_bol = (c == '\\n');"); + ("YY_CURRENT_BUFFER_LVALUE->yy_at_bol = (c == '\\n');"); if (do_yylineno) { indent_puts - ("if ( YY_G(yy_current_buffer)->yy_at_bol )"); + ("if ( YY_CURRENT_BUFFER_LVALUE->yy_at_bol )"); indent_up (); indent_puts ("++yylineno;"); indent_down (); @@ -405,6 +405,9 @@ void check_options () GEN_PREFIX ("_flush_buffer"); GEN_PREFIX ("_load_buffer_state"); GEN_PREFIX ("_switch_to_buffer"); + GEN_PREFIX ("push_buffer_state"); + GEN_PREFIX ("pop_buffer_state"); + GEN_PREFIX ("ensure_buffer_stack"); GEN_PREFIX ("lex"); GEN_PREFIX ("restart"); GEN_PREFIX ("lex_init"); @@ -753,6 +756,9 @@ void flexend (exit_status) "yy_set_bol", "yy_set_interactive", "yy_switch_to_buffer", + "yypush_buffer_state", + "yypop_buffer_state", + "yyensure_buffer_stack", "yyalloc", "yyconst", "yyextra", |