From 5218a435351f94ea39422e1ebe809d4bad1b365d Mon Sep 17 00:00:00 2001 From: Will Estes Date: Wed, 14 Nov 2001 21:13:06 +0000 Subject: more from the same batch --- TODO | 2 - flex.skl | 55 ++++--- flex.texi | 89 +++++++----- flexdef.h | 5 +- main.c | 232 ++++++++++++++++++++++++++++-- misc.c | 5 + tests/README | 2 + tests/TEMPLATE/Makefile.in | 2 +- tests/configure.in | 6 +- tests/test-basic-r/scanner.l | 2 +- tests/test-bison-yylloc/.cvsignore | 3 + tests/test-bison-yylloc/Makefile.in | 5 +- tests/test-bison-yylloc/parser.y | 18 --- tests/test-bison-yylloc/scanner.l | 2 +- tests/test-bison-yylval/.cvsignore | 3 + tests/test-bison-yylval/Makefile.in | 5 +- tests/test-bison-yylval/parser.y | 20 +-- tests/test-bison-yylval/scanner.l | 2 +- tests/test-include-by-reentrant/scanner.l | 4 +- tests/test-prefix-r/scanner.l | 28 ++-- tests/test-pthread/scanner.l | 8 +- tests/test-string-r/scanner.l | 2 +- tests/test-yyextra/scanner.l | 6 +- 23 files changed, 373 insertions(+), 133 deletions(-) diff --git a/TODO b/TODO index 61fed14..5715a1c 100644 --- a/TODO +++ b/TODO @@ -4,8 +4,6 @@ * the manual: -** include documentation of the new long options - ** integrate the items in the faqs/ directory into the manual or code or similar diff --git a/flex.skl b/flex.skl index 1b46d84..a3f330f 100644 --- a/flex.skl +++ b/flex.skl @@ -12,7 +12,7 @@ %# Each stop-point is numbered here and also in the code generator. %# (See gen.c, etc. for details.) %# %c - Begin linkage code that should NOT appear in a ".h" file. -%# %e - End linkage code. %c and %e are used +%# %e - End linkage code. %c and %e are used for building a header file. %# %# All control-lines EXCEPT comment lines ("%#") will be inserted into %# the generated scanner as a C-style comment. This is to aid those who @@ -97,17 +97,23 @@ #ifdef YY_REENTRANT +/* An opaque pointer. */ +#ifndef YY_TYPEDEF_YY_SCANNER_T +#define YY_TYPEDEF_YY_SCANNER_T +typedef void* yyscan_t; +#endif + /* For use wherever a Global is accessed or assigned. */ #define YY_G(var) (((struct yy_globals_t*)yy_globals)->var) /* For use in function prototypes to append the additional argument. */ #ifdef YY_USE_PROTOS -#define YY_LAST_ARG , void * yy_globals -#define YY_ONLY_ARG void * yy_globals +#define YY_LAST_ARG , yyscan_t yy_globals +#define YY_ONLY_ARG yyscan_t yy_globals #else #define YY_LAST_ARG , yy_globals #define YY_ONLY_ARG yy_globals -#define YY_DECL_LAST_ARG void * yy_globals; +#define YY_DECL_LAST_ARG yyscan_t yy_globals; #endif /* For use in function calls to pass the additional argument. */ @@ -123,8 +129,8 @@ #define yytext YY_G(yytext_r) #define yylineno YY_G(yylineno_r) -int yylex_init YY_PROTO((void** scanner)); -int yylex_destroy YY_PROTO((void* scanner)); +int yylex_init YY_PROTO((yyscan_t* scanner)); +int yylex_destroy YY_PROTO((yyscan_t scanner)); #else /* not YY_REENTRANT */ @@ -161,7 +167,11 @@ int yylex_destroy YY_PROTO((void* scanner)); /* Size of default input buffer. */ #define YY_BUF_SIZE 16384 + +#ifndef YY_TYPEDEF_YY_BUFFER_STATE +#define YY_TYPEDEF_YY_BUFFER_STATE typedef struct yy_buffer_state *YY_BUFFER_STATE; +#endif #ifndef YY_REENTRANT extern size_t yyleng; @@ -210,9 +220,14 @@ extern FILE *yyin, *yyout; * (without autoconf's help, which isn't available because we want * flex-generated scanners to compile on their own). */ -typedef unsigned int yy_size_t; +#ifndef YY_TYPEDEF_YY_SIZE_T +#define YY_TYPEDEF_YY_SIZE_T +typedef unsigned int yy_size_t; +#endif +#ifndef YY_STRUCT_YY_BUFFER_STATE +#define YY_STRUCT_YY_BUFFER_STATE struct yy_buffer_state { %- @@ -273,6 +288,7 @@ struct yy_buffer_state */ #define YY_BUFFER_EOF_PENDING 2 }; +#endif /* !YY_STRUCT_YY_BUFFER_STATE */ %- Standard (non-C++) definition %c @@ -315,17 +331,20 @@ static int yy_did_buffer_switch_on_eof; void yyrestart YY_PROTO(( FILE *input_file YY_LAST_ARG )); + void yy_switch_to_buffer YY_PROTO(( YY_BUFFER_STATE new_buffer YY_LAST_ARG )); void yy_load_buffer_state YY_PROTO(( YY_ONLY_ARG )); YY_BUFFER_STATE yy_create_buffer YY_PROTO(( FILE *file, int size YY_LAST_ARG )); void yy_delete_buffer YY_PROTO(( YY_BUFFER_STATE b YY_LAST_ARG )); void yy_init_buffer YY_PROTO(( YY_BUFFER_STATE b, FILE *file YY_LAST_ARG )); void yy_flush_buffer YY_PROTO(( YY_BUFFER_STATE b YY_LAST_ARG )); + #define YY_FLUSH_BUFFER yy_flush_buffer( YY_G(yy_current_buffer) YY_CALL_LAST_ARG) YY_BUFFER_STATE yy_scan_buffer YY_PROTO(( char *base, yy_size_t size YY_LAST_ARG )); YY_BUFFER_STATE yy_scan_string YY_PROTO(( yyconst char *yy_str YY_LAST_ARG )); YY_BUFFER_STATE yy_scan_bytes YY_PROTO(( yyconst char *bytes, int len YY_LAST_ARG )); + %* %c @@ -389,6 +408,7 @@ static void yy_fatal_error YY_PROTO(( yyconst char msg[] )); %- Reentrant structure and macros (non-C++). #ifdef YY_REENTRANT +%c struct yy_globals_t { @@ -432,9 +452,10 @@ struct yy_globals_t #endif }; +%e %c -static int yy_init_globals YY_PROTO(( void * )); +static int yy_init_globals YY_PROTO(( yyscan_t )); %e /* This must go here because YYSTYPE and YYLSTYPE are included @@ -444,7 +465,7 @@ static int yy_init_globals YY_PROTO(( void * )); # ifdef YYLTYPE # define yylloc YY_G(yylloc_r) # endif -#endif +#endif /* YY_REENTRANT_BISON_PURE */ #endif /* end if YY_REENTRANT */ @@ -1952,10 +1973,10 @@ void yyset_lloc( yyllocp YY_LAST_ARG ) #ifdef YY_USE_PROTOS -static int yy_init_globals( void* yy_globals) +static int yy_init_globals( yyscan_t yy_globals) #else static int yy_init_globals( yy_globals ) - void* yy_globals; + yyscan_t yy_globals; #endif { /* Initialization is the same as for the non-reentrant scanner. @@ -1986,22 +2007,22 @@ static int yy_init_globals( yy_globals ) /* User-visible API */ #ifdef YY_USE_PROTOS -int yylex_init( void** ptr_yy_globals) +int yylex_init( yyscan_t* ptr_yy_globals) #else int yylex_init( ptr_yy_globals ) - void** ptr_yy_globals; + yyscan_t* ptr_yy_globals; #endif { - *ptr_yy_globals = (void *) yy_flex_alloc ( sizeof( struct yy_globals_t ), NULL ); + *ptr_yy_globals = (yyscan_t) yy_flex_alloc ( sizeof( struct yy_globals_t ), NULL ); yy_init_globals ( *ptr_yy_globals ); return 0; } #ifdef YY_USE_PROTOS -int yylex_destroy( void* yy_globals ) +int yylex_destroy( yyscan_t yy_globals ) #else int yylex_destroy( yy_globals ) - void* yy_globals; + yyscan_t yy_globals; #endif { if( yy_globals ) @@ -2108,7 +2129,7 @@ int main() { #ifdef YY_REENTRANT - void *lexer; + yyscan_t lexer; yylex_init(&lexer); yylex( lexer ); yylex_destroy( lexer); diff --git a/flex.texi b/flex.texi index c7328ec..366ecce 100644 --- a/flex.texi +++ b/flex.texi @@ -3046,7 +3046,7 @@ another instance of itself. %% "eval(".+")" { - void * scanner; + yyscan_t scanner; yylex_init( &scanner ); yytext[yyleng-1] = ' '; @@ -3105,7 +3105,7 @@ First, an example of a reentrant scanner: %% int main ( int argc, char * argv[] ) { - void * scanner; + yyscan_t scanner; yylex_init ( &scanner ); yylex ( scanner ); @@ -3128,6 +3128,7 @@ Here are the things you need to do or know to use the reentrant C API of * init and destroy and destroy:: * accessor methods:: * extra data:: +* about yyscan_t:: @end menu @node specify reentrant, extra reentrant argument, reentrant detail, reentrant detail @@ -3154,15 +3155,16 @@ non-reentrant scanner. Here are the declarations of @example @verbatim - static void yy_push_state ( int new_state , void * yy_globals ) ; - static void yy_pop_state ( void * yy_globals ) ; + static void yy_push_state ( int new_state , yyscan_t yy_globals ) ; + static void yy_pop_state ( yyscan_t yy_globals ) ; @end verbatim @end example Notice that the argument @code{yy_globals} appears in the declaration of both functions. In fact, all @code{flex} functions in a reentrant scanner have this additional argument. It is always the last argument -in the argument list, it is always of type @code{void *}, and it is +in the argument list, it is always of type @code{yyscan_t} (which is +typedef'd to @code{void *}) and it is always named @code{yy_globals}. As you may have guessed, @code{yy_globals} is a pointer to an opaque data structure encapsulating the current state of the scanner. For a list of function declarations, @@ -3208,9 +3210,9 @@ after @code{yylex}, respectively. @example @verbatim - int yylex_init ( void ** ptr_yy_globals ) ; - int yylex ( void * yy_globals ) ; - int yylex_destroy ( void * yy_globals ) ; + int yylex_init ( yyscan_t * ptr_yy_globals ) ; + int yylex ( yyscan_t yy_globals ) ; + int yylex_destroy ( yyscan_t yy_globals ) ; @end verbatim @end example @@ -3251,11 +3253,11 @@ variable you want. For example: @example @verbatim /* Set the last character of yytext to NULL. */ - void chop ( void * scanner ) + void chop ( yyscan_t scanner ) { int len = yyget_leng( scanner ); yyget_text( scanner )[len - 1] = '\0'; - } + } @end verbatim @end example @@ -3268,7 +3270,7 @@ The above code may be called from within an action like this: @end verbatim @end example -@node extra data, , accessor methods, reentrant detail +@node extra data, about yyscan_t, accessor methods, reentrant detail @subsection Extra Data User-specific data can be stored in @code{yyextra}. @@ -3293,8 +3295,8 @@ from within the scanner itself. They are defined as follows: @example @verbatim #define YY_EXTRA_TYPE void* - YY_EXTRA_TYPE yyget_extra ( void * scanner ) ; - void yyset_extra ( YY_EXTRA_TYPE arbitrary_data , void * scanner) ; + YY_EXTRA_TYPE yyget_extra ( yyscan_t scanner ) ; + void yyset_extra ( YY_EXTRA_TYPE arbitrary_data , yyscan_t scanner) ; @end verbatim @end example @@ -3320,7 +3322,7 @@ defining @code{YY_EXTRA_TYPE} in section 1 of your scanner: %% void scan_file( char* filename ) { - void * scanner; + yyscan_t scanner; struct stat buf; yylex_init ( &scanner ); @@ -3334,6 +3336,24 @@ defining @code{YY_EXTRA_TYPE} in section 1 of your scanner: @end verbatim @end example + +@node about yyscan_t, , extra data, reentrant detail +@subsection About yyscan_t + +@code{yyscan_t} is defined as: + +@example +@verbatim + typedef void* yyscan_t; +@end verbatim +@end example + +It is initialized by @code{yylex_init()} to point to +an internal structure. You should never access this value +directly. In particular, you should never attempt to free it +(use @code{yylex_destroy()} instead.) + + @node bison pure, reentrant functions, reentrant detail, reentrant @section REENTRANT C SCANNERS WITH BISON PURE PARSERS @@ -3358,10 +3378,11 @@ specified, @code{flex} provides support for the functions @example @verbatim - YYSTYPE * yyget_lval ( void * scanner ) ; - void yyset_lval ( YYSTYPE * lvalp, void * scanner ); - YYLTYPE * yyget_lloc ( void * scanner ) ; - void yyset_lloc ( YYLTYPE * llocp, void * scanner ); + YYSTYPE * yyget_lval ( yyscan_t scanner ) ; + void yyset_lval ( YYSTYPE * lvalp, yyscan_t scanner ); + + YYLTYPE * yyget_lloc ( yyscan_t scanner ) ; + void yyset_lloc ( YYLTYPE * llocp, yyscan_t scanner ); @end verbatim @end example @@ -3369,8 +3390,8 @@ Accordingly, the declaration of yylex becomes one of the following: @example @verbatim - int yylex ( YYSTYPE * lvalp, void * scanner ); - int yylex ( YYSTYPE * lvalp, YYLTYPE * llocp, void * scanner ); + int yylex ( YYSTYPE * lvalp, yyscan_t scanner ); + int yylex ( YYSTYPE * lvalp, YYLTYPE * llocp, yyscan_t scanner ); @end verbatim @end example @@ -3436,16 +3457,16 @@ The following Functions are available in a reentrant scanner: @example @verbatim - char *yyget_text ( void * scanner ); - int yyget_leng ( void * scanner ); - FILE *yyget_in ( void * scanner ); - FILE *yyget_out ( void * scanner ); - int yyget_lineno ( void * scanner ); - YY_EXTRA_TYPE yyget_extra ( void * scanner ); - void yyset_in ( FILE * in_str , void * scanner ); - void yyset_out ( FILE * out_str , void * scanner ); - void yyset_lineno ( int line_number , void * scanner ); - void yyset_extra ( YY_EXTRA_TYPE user_defined , void * scanner ); + char *yyget_text ( yyscan_t scanner ); + int yyget_leng ( yyscan_t scanner ); + FILE *yyget_in ( yyscan_t scanner ); + FILE *yyget_out ( yyscan_t scanner ); + int yyget_lineno ( yyscan_t scanner ); + YY_EXTRA_TYPE yyget_extra ( yyscan_t scanner ); + void yyset_in ( FILE * in_str , yyscan_t scanner ); + void yyset_out ( FILE * out_str , yyscan_t scanner ); + void yyset_lineno ( int line_number , yyscan_t scanner ); + void yyset_extra ( YY_EXTRA_TYPE user_defined , yyscan_t scanner ); @end verbatim @end example @@ -3475,10 +3496,10 @@ reentrant-bison} (@samp{--reentrant-bison}) is specified: @example @verbatim - YYSTYPE * yyget_lval ( void * scanner ); - YYLTYPE *yyget_lloc ( void * scanner ); - void yyset_lval ( YYSTYPE * yylvalp , void * scanner ); - void yyset_lloc ( YYLTYPE * yyllocp , void * scanner ); + YYSTYPE * yyget_lval ( yyscan_t scanner ); + YYLTYPE *yyget_lloc ( yyscan_t scanner ); + void yyset_lval ( YYSTYPE * yylvalp , yyscan_t scanner ); + void yyset_lloc ( YYLTYPE * yyllocp , yyscan_t scanner ); yylval yylloc @end verbatim diff --git a/flexdef.h b/flexdef.h index e046288..3ec4909 100644 --- a/flexdef.h +++ b/flexdef.h @@ -1043,9 +1043,12 @@ extern struct Buf* buf_strappend PROTO((struct Buf*, const char* str)); extern struct Buf* buf_strnappend PROTO((struct Buf*, const char* str, int nchars)); extern struct Buf* buf_strdefine PROTO((struct Buf* buf, const char* str, const char* def)); -/* buffer for #define's generated by user-options on cmd line. */ +/* a string buffer for #define's generated by user-options on cmd line. */ extern struct Buf userdef_buf; +/* a char* buffer to save #define'd some symbols generated by flex. */ +extern struct Buf defs_buf; + /* For blocking out code from the header file. */ #define OUT_BEGIN_CODE() out_str("#ifndef %sIN_HEADER\n",prefix) #define OUT_END_CODE() out_str("#endif /* !%sIN_HEADER */\n",prefix); diff --git a/main.c b/main.c index b76e7ad..7b4c7c2 100644 --- a/main.c +++ b/main.c @@ -326,6 +326,12 @@ void check_options() GEN_PREFIX( "FlexLexer" ); else { + outn( "#ifndef YY_REENTRANT" ); + GEN_PREFIX( "text" ); + GEN_PREFIX( "leng" ); + GEN_PREFIX( "in" ); + GEN_PREFIX( "out" ); + outn( "#endif" ); GEN_PREFIX( "_create_buffer" ); GEN_PREFIX( "_delete_buffer" ); GEN_PREFIX( "_scan_buffer" ); @@ -364,6 +370,9 @@ void check_options() outn ( "#define YY_USE_LINENO 1"); } + if ( do_yylineno && !reentrant ) + GEN_PREFIX( "lineno" ); + if ( do_yywrap ) GEN_PREFIX( "wrap" ); @@ -381,6 +390,59 @@ void check_options() /* %% [1.0] */ } +/* Alter #line directives from the generated source, destined + * for the generated header. We chaneg the line number and filename. + * linebuf is modified in place. + */ +void fix_line_dirs(linebuf, outfilename, headerfilename, nlines) +char * linebuf; +char * outfilename; +char * headerfilename; +int nlines; +{ + char *pname, *p; + /* Match pattern: ^#line +[:digit:]+ +"outfilename" */ + p = linebuf; + if (strncmp(p, "#line ", 6)) + return; + p += 6; + + /* match spaces */ + while (*p == ' ') + p++; + if (!isdigit(*p)) + return; + + /* match numbers */ + while (isdigit(*p)) + p++; + if (*p != ' ') + return; + + /* match spaces */ + while (*p == ' ') + p++; + if (*p != '"') + return; + p++; + + pname = p; + + /* find end of filename. Note: If scanner filename has + * embedded '"' chars, then the generated #line directive + * may fail. */ + while (*p != '\0' && *p != '"') + p++; + if (*p != '"') + return; + + if (strncmp(pname, outfilename, p-pname) != 0) + return; + + /* We have a match. */ + + sprintf(linebuf,"#line %d \"%s\"\n", nlines+2, headerfilename); +} /* flexend - terminate flex * @@ -394,6 +456,7 @@ int exit_status; { static int called_before = -1; /* prevent infinite recursion. */ int tblsiz; + int i; int unlink(); if( ++called_before ) @@ -413,10 +476,10 @@ int exit_status; if ( headerfilename && exit_status == 0 && outfile_created && !ferror(stdout)) { /* Copy the file we just wrote to a header file. */ - #define COPY_SZ 512 + #define LINE_SZ 512 FILE *header_out; - char copybuf[COPY_SZ]; - int ncopy; + char linebuf[LINE_SZ]; + int nlines=0; /* rewind the outfile file. */ fflush(stdout); @@ -433,11 +496,155 @@ int exit_status; prefix,prefix,prefix); fflush(header_out); - while((ncopy=fread(copybuf, 1, COPY_SZ, stdout)) > 0) - if ( fwrite(copybuf, 1, ncopy, header_out) <= 0) - break; + nlines=4; + while(fgets(linebuf, LINE_SZ, stdout)) { + fix_line_dirs(linebuf, outfilename, headerfilename, nlines); + fputs(linebuf, header_out); + nlines++; + } + + /* Kill ALL flex-related macros. This is so the user + * can #include more than one generated header file. */ + fprintf(header_out,"\n"); + fprintf(header_out,"#undef BEGIN\n"); + fprintf(header_out,"#undef ECHO\n"); + fprintf(header_out,"#undef EOB_ACT_CONTINUE_SCAN\n"); + fprintf(header_out,"#undef EOB_ACT_END_OF_FILE\n"); + fprintf(header_out,"#undef EOB_ACT_LAST_MATCH\n"); + fprintf(header_out,"#undef FLEX_SCANNER\n"); + fprintf(header_out,"#undef FLEX_STD\n"); + fprintf(header_out,"#undef INITIAL\n"); + fprintf(header_out,"#undef REJECT\n"); + fprintf(header_out,"#undef YYLMAX\n"); + fprintf(header_out,"#undef YYSTATE\n"); + fprintf(header_out,"#undef YY_AT_BOL\n"); + fprintf(header_out,"#undef YY_BREAK\n"); + fprintf(header_out,"#undef YY_BUFFER_EOF_PENDING\n"); + fprintf(header_out,"#undef YY_BUFFER_NEW\n"); + fprintf(header_out,"#undef YY_BUFFER_NORMAL\n"); + fprintf(header_out,"#undef YY_BUF_SIZE\n"); + fprintf(header_out,"#undef YY_CALL_LAST_ARG\n"); + fprintf(header_out,"#undef YY_CALL_ONLY_ARG\n"); + fprintf(header_out,"#undef YY_CURRENT_BUFFER\n"); + fprintf(header_out,"#undef YY_DECL\n"); + fprintf(header_out,"#undef YY_DECL_LAST_ARG\n"); + fprintf(header_out,"#undef YY_DO_BEFORE_ACTION\n"); + fprintf(header_out,"#undef YY_END_OF_BUFFER\n"); + fprintf(header_out,"#undef YY_END_OF_BUFFER_CHAR\n"); + fprintf(header_out,"#undef YY_EXIT_FAILURE\n"); + fprintf(header_out,"#undef YY_EXTRA_TYPE\n"); + fprintf(header_out,"#undef YY_FATAL_ERROR\n"); + fprintf(header_out,"#undef YY_FLEX_DEFINED_ECHO\n"); + fprintf(header_out,"#undef YY_FLEX_LEX_COMPAT\n"); + fprintf(header_out,"#undef YY_FLEX_MAJOR_VERSION\n"); + fprintf(header_out,"#undef YY_FLEX_MINOR_VERSION\n"); + fprintf(header_out,"#undef YY_FLUSH_BUFFER\n"); + fprintf(header_out,"#undef YY_G\n"); + fprintf(header_out,"#undef YY_INPUT\n"); + fprintf(header_out,"#undef YY_INTERACTIVE\n"); + fprintf(header_out,"#undef YY_LAST_ARG\n"); + fprintf(header_out,"#undef YY_LEX_ARGS\n"); + fprintf(header_out,"#undef YY_MAIN\n"); + fprintf(header_out,"#undef YY_MORE_ADJ\n"); + fprintf(header_out,"#undef YY_NEED_STRLEN\n"); + fprintf(header_out,"#undef YY_NEW_FILE\n"); + fprintf(header_out,"#undef YY_NO_GET_EXTRA\n"); + fprintf(header_out,"#undef YY_NO_GET_IN\n"); + fprintf(header_out,"#undef YY_NO_GET_LENG\n"); + fprintf(header_out,"#undef YY_NO_GET_LINENO\n"); + fprintf(header_out,"#undef YY_NO_GET_LLOC\n"); + fprintf(header_out,"#undef YY_NO_GET_LVAL\n"); + fprintf(header_out,"#undef YY_NO_GET_OUT\n"); + fprintf(header_out,"#undef YY_NO_GET_TEXT\n"); + fprintf(header_out,"#undef YY_NO_INPUT\n"); + fprintf(header_out,"#undef YY_NO_POP_STATE\n"); + fprintf(header_out,"#undef YY_NO_PUSH_STATE\n"); + fprintf(header_out,"#undef YY_NO_SCAN_BUFFER\n"); + fprintf(header_out,"#undef YY_NO_SCAN_BYTES\n"); + fprintf(header_out,"#undef YY_NO_SCAN_STRING\n"); + fprintf(header_out,"#undef YY_NO_SET_EXTRA\n"); + fprintf(header_out,"#undef YY_NO_SET_IN\n"); + fprintf(header_out,"#undef YY_NO_SET_LINENO\n"); + fprintf(header_out,"#undef YY_NO_SET_LLOC\n"); + fprintf(header_out,"#undef YY_NO_SET_LVAL\n"); + fprintf(header_out,"#undef YY_NO_SET_OUT\n"); + fprintf(header_out,"#undef YY_NO_TOP_STATE\n"); + fprintf(header_out,"#undef YY_NO_UNPUT\n"); + fprintf(header_out,"#undef YY_NULL\n"); + fprintf(header_out,"#undef YY_NUM_RULES\n"); + fprintf(header_out,"#undef YY_ONLY_ARG\n"); + fprintf(header_out,"#undef YY_PROTO\n"); + fprintf(header_out,"#undef YY_READ_BUF_SIZE\n"); + fprintf(header_out,"#undef YY_REENTRANT\n"); + fprintf(header_out,"#undef YY_REENTRANT_BISON_PURE\n"); + fprintf(header_out,"#undef YY_RESTORE_YY_MORE_OFFSET\n"); + fprintf(header_out,"#undef YY_RULE_SETUP\n"); + fprintf(header_out,"#undef YY_SC_TO_UI\n"); + fprintf(header_out,"#undef YY_SKIP_YYWRAP\n"); + fprintf(header_out,"#undef YY_STACK_USED\n"); + fprintf(header_out,"#undef YY_START\n"); + fprintf(header_out,"#undef YY_START_STACK_INCR\n"); + fprintf(header_out,"#undef YY_STATE_EOF\n"); + fprintf(header_out,"#undef YY_STDINIT\n"); + fprintf(header_out,"#undef YY_TEXT_IS_ARRAY\n"); + fprintf(header_out,"#undef YY_TRAILING_HEAD_MASK\n"); + fprintf(header_out,"#undef YY_TRAILING_MASK\n"); + fprintf(header_out,"#undef YY_USER_ACTION\n"); + fprintf(header_out,"#undef YY_USES_REJECT\n"); + fprintf(header_out,"#undef YY_USE_CONST\n"); + fprintf(header_out,"#undef YY_USE_LINENO\n"); + fprintf(header_out,"#undef YY_USE_PROTOS\n"); + fprintf(header_out,"#undef unput\n"); + fprintf(header_out,"#undef yy_create_buffer\n"); + fprintf(header_out,"#undef yy_delete_buffer\n"); + fprintf(header_out,"#undef yy_flex_debug\n"); + fprintf(header_out,"#undef yy_flush_buffer\n"); + fprintf(header_out,"#undef yy_init_buffer\n"); + fprintf(header_out,"#undef yy_load_buffer_state\n"); + fprintf(header_out,"#undef yy_new_buffer\n"); + fprintf(header_out,"#undef yy_scan_buffer\n"); + fprintf(header_out,"#undef yy_scan_bytes\n"); + fprintf(header_out,"#undef yy_scan_string\n"); + fprintf(header_out,"#undef yy_set_bol\n"); + fprintf(header_out,"#undef yy_set_interactive\n"); + fprintf(header_out,"#undef yy_switch_to_buffer\n"); + fprintf(header_out,"#undef yyconst\n"); + fprintf(header_out,"#undef yyextra\n"); + fprintf(header_out,"#undef yyget_extra\n"); + fprintf(header_out,"#undef yyget_in\n"); + fprintf(header_out,"#undef yyget_leng\n"); + fprintf(header_out,"#undef yyget_lineno\n"); + fprintf(header_out,"#undef yyget_lloc\n"); + fprintf(header_out,"#undef yyget_lval\n"); + fprintf(header_out,"#undef yyget_out\n"); + fprintf(header_out,"#undef yyget_text\n"); + fprintf(header_out,"#undef yyin\n"); + fprintf(header_out,"#undef yyleng\n"); + fprintf(header_out,"#undef yyless\n"); + fprintf(header_out,"#undef yylex\n"); + fprintf(header_out,"#undef yylex_destroy\n"); + fprintf(header_out,"#undef yylex_init\n"); + fprintf(header_out,"#undef yylineno\n"); + fprintf(header_out,"#undef yylloc\n"); + fprintf(header_out,"#undef yylval\n"); + fprintf(header_out,"#undef yymore\n"); + fprintf(header_out,"#undef yyout\n"); + fprintf(header_out,"#undef yyrestart\n"); + fprintf(header_out,"#undef yyset_extra\n"); + fprintf(header_out,"#undef yyset_in\n"); + fprintf(header_out,"#undef yyset_lineno\n"); + fprintf(header_out,"#undef yyset_lloc\n"); + fprintf(header_out,"#undef yyset_lval\n"); + fprintf(header_out,"#undef yyset_out\n"); + fprintf(header_out,"#undef yyterminate\n"); + fprintf(header_out,"#undef yytext\n"); + fprintf(header_out,"#undef yytext_ptr\n"); + fprintf(header_out,"#undef yywrap\n"); + + /* undef any of the auto-generated symbols. */ + for(i=0; i < defs_buf.nelts; i++) + fprintf(header_out, "#undef %s\n", ((char**)defs_buf.elts)[i]); - fflush(header_out); fprintf(header_out, "\n" "#undef %sIN_HEADER\n" @@ -446,6 +653,7 @@ int exit_status; if ( ferror( header_out ) ) lerrsf( _( "error creating header file %s" ), headerfilename); + fflush(header_out); fclose(header_out); } @@ -701,8 +909,10 @@ char **argv; defs1_offset = prolog_offset = action_offset = action_index = 0; action_array[0] = '\0'; - /* Initialize any buffers. */ - buf_init(&userdef_buf, sizeof(char)); + /* Initialize any buffers. */ + buf_init(&userdef_buf, sizeof(char)); + buf_init(&defs_buf, sizeof(char*)); + /* Enable C++ if program name ends with '+'. */ program_name = basename2(argv[0],0); @@ -1234,10 +1444,12 @@ _( "Variable trailing context rules entail a large performance penalty\n" ) ); if ( ddebug ) outn( "\n#define FLEX_DEBUG" ); + OUT_BEGIN_CODE(); if ( csize == 256 ) outn( "typedef unsigned char YY_CHAR;" ); else outn( "typedef char YY_CHAR;" ); + OUT_END_CODE(); if ( C_plus_plus ) { @@ -1283,10 +1495,12 @@ _( "Variable trailing context rules entail a large performance penalty\n" ) ); OUT_END_CODE(); } + OUT_BEGIN_CODE(); if ( fullspd ) outn( "typedef yyconst struct yy_trans_info *yy_state_type;" ); else if ( ! C_plus_plus ) outn( "typedef int yy_state_type;" ); + OUT_END_CODE(); if ( ddebug ) outn( "\n#define FLEX_DEBUG" ); diff --git a/misc.c b/misc.c index 3c37e33..967a413 100644 --- a/misc.c +++ b/misc.c @@ -40,6 +40,7 @@ char *defname; int value; { char buf[MAXLINE]; + char *cpy; if ( (int) strlen( defname ) > MAXLINE / 2 ) { @@ -50,6 +51,10 @@ int value; sprintf( buf, "#define %s %d\n", defname, value ); add_action( buf ); + + /* track #defines so we can undef them when we're done. */ + cpy = copy_string(defname); + buf_append(&defs_buf, &cpy, 1); } diff --git a/tests/README b/tests/README index 6b55ca0..031cc99 100644 --- a/tests/README +++ b/tests/README @@ -66,6 +66,8 @@ DESCRIPTION OF TESTS header-r - Test generated header file, reentrant. include-by-buffer - YY_BUFFER_STATE, yy_push_state, etc. include-by-reentrant - Nested scanners. + multiple-scanners-nr - #include and run two separate scanners, non-reentrant. + multiple-scanners-r - #include and run two separate scanners, reentrant. prefix-nr - verify prefixes are working, nonreentrant. prefix-r - verify prefixes are working, reentrant. pthread - pthreads test. A NO-OP if libpthread not found. diff --git a/tests/TEMPLATE/Makefile.in b/tests/TEMPLATE/Makefile.in index 67b1542..1a8260c 100644 --- a/tests/TEMPLATE/Makefile.in +++ b/tests/TEMPLATE/Makefile.in @@ -61,5 +61,5 @@ distclean: clean rm -f Makefile clean: - rm -f scanner.o scanner.c parser.o parser.c parser.h parser.h $(TESTNAME) + rm -f scanner.o scanner.c parser.o parser.c parser.h parser.h $(TESTNAME) OUTPUT diff --git a/tests/configure.in b/tests/configure.in index 83373c5..1bd4d62 100644 --- a/tests/configure.in +++ b/tests/configure.in @@ -42,7 +42,8 @@ TESTDIRS="test-string-r test-pthread \ TEMPLATE test-basic-nr test-basic-r \ test-include-by-buffer test-string-nr \ test-bison-yylloc test-yyextra test-prefix-nr \ - test-prefix-r test-header-nr test-header-r" + test-prefix-r test-header-nr test-header-r \ + test-multiple-scanners-nr test-multiple-scanners-r" AC_OUTPUT( Makefile test-string-r/Makefile test-pthread/Makefile test-bison-yylval/Makefile test-include-by-reentrant/Makefile @@ -50,5 +51,6 @@ AC_OUTPUT( Makefile test-string-r/Makefile test-pthread/Makefile test-include-by-buffer/Makefile test-string-nr/Makefile test-bison-yylloc/Makefile test-yyextra/Makefile test-prefix-nr/Makefile test-prefix-r/Makefile - test-header-nr/Makefile test-header-r/Makefile) + test-header-nr/Makefile test-header-r/Makefile + test-multiple-scanners-nr/Makefile test-multiple-scanners-r/Makefile) diff --git a/tests/test-basic-r/scanner.l b/tests/test-basic-r/scanner.l index a31a76d..cc3d87a 100644 --- a/tests/test-basic-r/scanner.l +++ b/tests/test-basic-r/scanner.l @@ -29,7 +29,7 @@ WS [[:blank:]] int main (int argc, char** argv ) { - void * lexer; + yyscan_t lexer; yylex_init( &lexer ); yyset_out ( stdout,lexer); yyset_in ( stdin, lexer); diff --git a/tests/test-bison-yylloc/.cvsignore b/tests/test-bison-yylloc/.cvsignore index c88d7ef..b99373d 100644 --- a/tests/test-bison-yylloc/.cvsignore +++ b/tests/test-bison-yylloc/.cvsignore @@ -2,4 +2,7 @@ Makefile parser.c parser.h scanner.c +scanner.h +yyheader.h test-bison-yylloc + diff --git a/tests/test-bison-yylloc/Makefile.in b/tests/test-bison-yylloc/Makefile.in index 99c65a9..d3b933b 100644 --- a/tests/test-bison-yylloc/Makefile.in +++ b/tests/test-bison-yylloc/Makefile.in @@ -42,7 +42,8 @@ $(TESTNAME): parser.y scanner.l $(YACC) $(YFLAGS) $(srcdir)/parser.y ; \ $(CC) $(CPPFLAGS) $(CFLAGS) -c scanner.c ; \ $(CC) $(CPPFLAGS) $(CFLAGS) -c parser.c ; \ - $(CC) $(CFLAGS) -o $(TESTNAME) scanner.o parser.o $(LDFLAGS) $(LIBS) ; \ + $(CC) $(CPPFLAGS) $(CFLAGS) -c $(srcdir)/main.c ; \ + $(CC) $(CFLAGS) -o $(TESTNAME) scanner.o parser.o main.o $(LDFLAGS) $(LIBS) ; \ fi test: check @@ -57,7 +58,7 @@ check: $(TESTNAME) fi clean: - rm -f scanner.o scanner.c parser.o parser.c parser.h $(TESTNAME) + rm -f scanner.o scanner.c main.o parser.o parser.c parser.h $(TESTNAME) yyheader.h distclean: clean diff --git a/tests/test-bison-yylloc/parser.y b/tests/test-bison-yylloc/parser.y index d545151..bcddcde 100644 --- a/tests/test-bison-yylloc/parser.y +++ b/tests/test-bison-yylloc/parser.y @@ -12,13 +12,7 @@ #define YYPARSE_PARAM scanner #define YYLEX_PARAM scanner - int yyerror(char* msg); -extern int testlex_init(void**); -extern int testlex_destroy(void**); -extern int testset_in(FILE*,void*); -extern int testlex(); -extern int testget_lineno(void*); /* A dummy function. A check against seg-faults in yylval->str. */ int process_text(char* s) { @@ -75,15 +69,3 @@ int yyerror(char* msg) { return 0; } - -int main ( int argc, char** argv ) -{ - void* scanner; - /*yydebug =1;*/ - testlex_init ( &scanner ); - testset_in(stdin,scanner); - testparse ( scanner ); - testlex_destroy ( scanner ); - return 0; -} - diff --git a/tests/test-bison-yylloc/scanner.l b/tests/test-bison-yylloc/scanner.l index 0c5ef00..a5724ed 100644 --- a/tests/test-bison-yylloc/scanner.l +++ b/tests/test-bison-yylloc/scanner.l @@ -11,7 +11,7 @@ static char* STRDUP(char* s1); %option 8bit outfile="scanner.c" prefix="test" %option reentrant-bison yylineno %option nomain nounput noyy_top_state noyywrap nodefault warn -%option prefix="test" +%option prefix="test" header="scanner.h" %% diff --git a/tests/test-bison-yylval/.cvsignore b/tests/test-bison-yylval/.cvsignore index 11b8464..a9aa90c 100644 --- a/tests/test-bison-yylval/.cvsignore +++ b/tests/test-bison-yylval/.cvsignore @@ -2,4 +2,7 @@ Makefile parser.c parser.h scanner.c +scanner.h +yyheader.h test-bison-yylval + diff --git a/tests/test-bison-yylval/Makefile.in b/tests/test-bison-yylval/Makefile.in index b2507f1..c3636ad 100644 --- a/tests/test-bison-yylval/Makefile.in +++ b/tests/test-bison-yylval/Makefile.in @@ -42,7 +42,8 @@ $(TESTNAME): parser.y scanner.l $(YACC) $(YFLAGS) $(srcdir)/parser.y ; \ $(CC) $(CPPFLAGS) $(CFLAGS) -c scanner.c ;\ $(CC) $(CPPFLAGS) $(CFLAGS) -c parser.c ; \ - $(CC) $(CFLAGS) -o $(TESTNAME) scanner.o parser.o $(LDFLAGS) $(LIBS) ; \ + $(CC) $(CPPFLAGS) $(CFLAGS) -c $(srcdir)/main.c ; \ + $(CC) $(CFLAGS) -o $(TESTNAME) main.o scanner.o parser.o $(LDFLAGS) $(LIBS) ; \ fi test: check @@ -56,7 +57,7 @@ check: $(TESTNAME) fi clean: - rm -f scanner.o scanner.c parser.o parser.c parser.h $(TESTNAME) + rm -f scanner.o scanner.c parser.o parser.c main.o parser.h $(TESTNAME) yyheader.h distclean: clean diff --git a/tests/test-bison-yylval/parser.y b/tests/test-bison-yylval/parser.y index 8dfebd8..1e9517a 100644 --- a/tests/test-bison-yylval/parser.y +++ b/tests/test-bison-yylval/parser.y @@ -12,12 +12,8 @@ #define YYPARSE_PARAM scanner #define YYLEX_PARAM scanner - int yyerror(char* msg); -extern int testlex_init(void**); -extern int testlex_destroy(void**); -extern int testset_in(FILE*,void*); -extern int testlex(); + /* A dummy function. A check against seg-faults in yylval->str. */ int process_text(char* s) { @@ -38,6 +34,7 @@ int process_text(char* s) { long unused; char * str; } + %token TAGNAME TEXT %token LT %token GT @@ -60,16 +57,3 @@ int yyerror(char* msg) { fprintf(stderr,"%s\n",msg); return 0; } - - -int main ( int argc, char** argv ) -{ - void* scanner; - /*yydebug =1;*/ - testlex_init ( &scanner ); - testset_in(stdin,scanner); - testparse ( scanner ); - testlex_destroy ( scanner ); - return 0; -} - diff --git a/tests/test-bison-yylval/scanner.l b/tests/test-bison-yylval/scanner.l index d2c7981..8cd3adb 100644 --- a/tests/test-bison-yylval/scanner.l +++ b/tests/test-bison-yylval/scanner.l @@ -13,7 +13,7 @@ enum yesno_t { no=0, yes=1 }; %option 8bit outfile="scanner.c" prefix="test" %option reentrant-bison %option noyywrap nomain nounput noyy_top_state noyywrap nodefault warn -%option prefix="test" +%option prefix="test" header="scanner.h" %option stack diff --git a/tests/test-include-by-reentrant/scanner.l b/tests/test-include-by-reentrant/scanner.l index 80aceff..be43b3a 100644 --- a/tests/test-include-by-reentrant/scanner.l +++ b/tests/test-include-by-reentrant/scanner.l @@ -25,7 +25,7 @@ { [[:alnum:]_.-]+> { /* recurse */ - void * scanner; + yyscan_t scanner; FILE * fp; yytext[yyleng-1]='\0'; if((fp=fopen(yytext,"r"))==NULL) { @@ -55,7 +55,7 @@ int main ( int argc, char** argv ) { FILE * fp; - void * scanner; + yyscan_t scanner; if( argc != 2 ) { fprintf(stderr,"*** Error: Must specifiy one filename.\n"); exit(-1); diff --git a/tests/test-prefix-r/scanner.l b/tests/test-prefix-r/scanner.l index 5c3b45a..b091d9f 100644 --- a/tests/test-prefix-r/scanner.l +++ b/tests/test-prefix-r/scanner.l @@ -28,19 +28,19 @@ FOO_scan_bytes( (yyconst char*)0, 0, yy_globals); FOO_scan_string( (yyconst char*)0, yy_globals); FOO_switch_to_buffer( (YY_BUFFER_STATE)0, yy_globals); - FOOrestart( (FILE*)0, (void*)0); - - FOOget_extra( (void*)0 ); - FOOget_in( (void*)0 ); - FOOget_leng( (void*)0 ); - FOOget_out( (void*)0 ); - FOOget_text( (void*)0 ); - FOOlex( (void*)0 ); - FOOlex_destroy( (void*)0 ); - FOOlex_init( (void**)0 ); - FOOset_extra( (void*)0, (void*)0 ); - FOOset_in( (FILE*)0, (void*)0 ); - FOOset_out( (FILE*)0, (void*)0 ); + FOOrestart( (FILE*)0, (yyscan_t )0); + + FOOget_extra( (yyscan_t )0 ); + FOOget_in( (yyscan_t )0 ); + FOOget_leng( (yyscan_t )0 ); + FOOget_out( (yyscan_t )0 ); + FOOget_text( (yyscan_t )0 ); + FOOlex( (yyscan_t )0 ); + FOOlex_destroy( (yyscan_t )0 ); + FOOlex_init( (yyscan_t *)0 ); + FOOset_extra( (void *)0, (yyscan_t )0 ); + FOOset_in( (FILE*)0, (yyscan_t )0 ); + FOOset_out( (FILE*)0, (yyscan_t )0 ); } } %% @@ -48,7 +48,7 @@ int main ( int argc, char** argv ) { - void * scanner; + yyscan_t scanner; FOOlex_init( &scanner); FOOlex( scanner); FOOlex_destroy( scanner); diff --git a/tests/test-pthread/scanner.l b/tests/test-pthread/scanner.l index 55d9274..22ca84e 100644 --- a/tests/test-pthread/scanner.l +++ b/tests/test-pthread/scanner.l @@ -19,7 +19,7 @@ #endif /* A naive test for segfaults when accessing yytext. */ -static int process_text(char* s, void* scanner); +static int process_text(char* s, yyscan_t scanner); %} @@ -55,8 +55,8 @@ static int process_text(char* s, void* scanner); [[:space:]\r\n]+ { } %% -int yywrap( void * scanner) { return 1; } -static int process_text(char* s, void* scanner) +int yywrap( yyscan_t scanner) { return 1; } +static int process_text(char* s, yyscan_t scanner) { return (int)(*s) + (int) *(s + yyget_leng(scanner)-1); } @@ -96,7 +96,7 @@ void * thread_func ( void* arg ) for( i =0 ; i < N_SCANS ; i++ ) { int next; - void * scanner; + yyscan_t scanner; FILE * fp; pthread_mutex_lock ( &next_lock ); diff --git a/tests/test-string-r/scanner.l b/tests/test-string-r/scanner.l index eb02e76..fefa396 100644 --- a/tests/test-string-r/scanner.l +++ b/tests/test-string-r/scanner.l @@ -39,7 +39,7 @@ main ( int argc, char** argv ) char * buf; int len; YY_BUFFER_STATE state; - void * scanner=NULL; + yyscan_t scanner=NULL; /* Scan a good string. */ diff --git a/tests/test-yyextra/scanner.l b/tests/test-yyextra/scanner.l index b112c21..f0834e1 100644 --- a/tests/test-yyextra/scanner.l +++ b/tests/test-yyextra/scanner.l @@ -20,7 +20,7 @@ struct Buffer { #define YY_EXTRA_TYPE struct Buffer * /* Save char into junk array at next position. */ -static void append_char (char c, void* scanner ); +static void append_char (char c, yyscan_t scanner ); %} @@ -39,7 +39,7 @@ static void append_char (char c, void* scanner ); int main ( int argc, char** argv ) { - void * scanner; + yyscan_t scanner; struct Buffer * buf; int i; @@ -66,7 +66,7 @@ main ( int argc, char** argv ) } /* Save char into junk array at next position. */ -static void append_char (char c, void* scanner ) +static void append_char (char c, yyscan_t scanner ) { struct Buffer *buf, *new_buf; buf = testget_extra(scanner); -- cgit v1.2.3