diff options
| -rw-r--r-- | ccl.c | 7 | ||||
| -rw-r--r-- | flex.skl | 5 | ||||
| -rw-r--r-- | flexdef.h | 13 | ||||
| -rw-r--r-- | gen.c | 66 | ||||
| -rw-r--r-- | main.c | 27 | ||||
| -rw-r--r-- | nfa.c | 18 | ||||
| -rw-r--r-- | parse.y | 49 | ||||
| -rw-r--r-- | scan.l | 2 |
8 files changed, 128 insertions, 59 deletions
@@ -52,6 +52,10 @@ int ch; if ( ccltbl[ind + i] == ch ) return; + /* mark newlines */ + if (ch == nlch) + ccl_has_nl[cclp] = true; + newpos = ind + len; if ( newpos >= current_max_ccl_tbl_size ) @@ -82,6 +86,7 @@ int cclinit() cclmap = reallocate_integer_array( cclmap, current_maxccls ); ccllen = reallocate_integer_array( ccllen, current_maxccls ); cclng = reallocate_integer_array( cclng, current_maxccls ); + ccl_has_nl = reallocate_bool_array( ccl_has_nl, current_maxccls ); } if ( lastccl == 1 ) @@ -98,6 +103,7 @@ int cclinit() ccllen[lastccl] = 0; cclng[lastccl] = 0; /* ccl's start out life un-negated */ + ccl_has_nl[lastccl] = false; return lastccl; } @@ -109,6 +115,7 @@ void cclnegate( cclp ) int cclp; { cclng[cclp] = 1; + ccl_has_nl[cclp] = !ccl_has_nl[cclp]; } @@ -1283,7 +1283,10 @@ void yyFlexLexer::yyunput( int c, register char* yy_bp) *--yy_cp = (char) c; %% [18.0] update yylineno here - +#ifdef YY_USE_LINENO + if ( c == '\n' ) + --yylineno; +#endif YY_G(yytext_ptr) = yy_bp; YY_G(yy_hold_char) = *yy_cp; YY_G(yy_c_buf_p) = yy_cp; @@ -477,6 +477,9 @@ extern int onenext[ONE_STACK_SIZE], onedef[ONE_STACK_SIZE], onesp; * context * rule_linenum - line number associated with rule * rule_useful - true if we've determined that the rule can be matched + * rule_has_nl - true if rule could possibly match a newline + * ccl_has_nl - true if current ccl could match a newline + * nlch - default eol char */ extern int maximum_mns, current_mns, current_max_rules; @@ -484,6 +487,8 @@ extern int num_rules, num_eof_rules, default_rule, lastnfa; extern int *firstst, *lastst, *finalst, *transchar, *trans1, *trans2; extern int *accptnum, *assoc_rule, *state_type; extern int *rule_type, *rule_linenum, *rule_useful; +extern bool *rule_has_nl, *ccl_has_nl; +extern int nlch; /* Different types of states; values are useful as masks, as well, for * routines like check_trailing_context(). @@ -659,6 +664,12 @@ void flex_free PROTO((void*)); #define reallocate_integer_array(array,size) \ (int *) reallocate_array( (void *) array, size, sizeof( int ) ) +#define allocate_bool_array(size) \ + (bool *) allocate_array( size, sizeof( bool ) ) + +#define reallocate_bool_array(array,size) \ + (bool *) reallocate_array( (void *) array, size, sizeof( bool ) ) + #define allocate_int_ptr_array(size) \ (int **) allocate_array( size, sizeof( int * ) ) @@ -919,7 +930,7 @@ extern int copysingl PROTO((int, int)); extern void dumpnfa PROTO((int)); /* Finish up the processing for a rule. */ -extern void finish_rule PROTO((int, int, int, int)); +extern void finish_rule PROTO((int, int, int, int, int)); /* Connect two machines together. */ extern int link_machines PROTO((int, int)); @@ -79,6 +79,26 @@ void do_indent() } } +/* Generate the table for possible eol matches. */ +static void geneoltbl() +{ + int i; + + outn("#ifdef YY_USE_LINENO" ); + outn( "/* Table of booleans, true if rule could match eol. */" ); + out_dec("static const int yy_rule_can_match_eol[%d] =\n {\n ", + num_rules + 1); + + for (i = 0; i < num_rules; i++){ + out_dec("%d, ", rule_has_nl[i]?1:0); + /* format nicely, 20 numbers per line. */ + if ((i%20) == 19) + out("\n "); + } + out_dec("%d\n };\n", rule_has_nl[i]?1:0); + outn( "#endif" ); +} + /* Generate the code to keep backing-up information. */ @@ -214,7 +234,7 @@ void genctbl() out_dec( "static yyconst struct yy_trans_info *yy_start_state_list[%d] =\n", lastsc * 2 + 1 ); - outn( " {" ); /* } so vi doesn't get confused */ + outn( " {" ); for ( i = 0; i <= lastsc * 2; ++i ) out_dec( " &yy_transition[%d],\n", base[i] ); @@ -512,7 +532,7 @@ void gen_next_match() if ( num_backing_up > 0 ) { - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); gen_backing_up(); outc( '\n' ); } @@ -520,7 +540,7 @@ void gen_next_match() indent_puts( "++yy_cp;" ); if ( num_backing_up > 0 ) - /* { for vi */ + indent_puts( "}" ); indent_down(); @@ -531,7 +551,7 @@ void gen_next_match() else if ( fullspd ) { - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); indent_puts( "register yyconst struct yy_trans_info *yy_trans_info;\n" ); indent_puts( "register YY_CHAR yy_c;\n" ); @@ -544,18 +564,18 @@ void gen_next_match() indent_up(); if ( num_backing_up > 0 ) - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); indent_puts( "yy_current_state += yy_trans_info->yy_nxt;" ); if ( num_backing_up > 0 ) { outc( '\n' ); - gen_backing_up(); /* { for vi */ + gen_backing_up(); indent_puts( "}" ); } - indent_down(); /* { for vi */ + indent_down(); indent_puts( "}" ); } @@ -564,13 +584,13 @@ void gen_next_match() indent_puts( "do" ); indent_up(); - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); gen_next_state( false ); indent_puts( "++yy_cp;" ); - /* { for vi */ + indent_puts( "}" ); indent_down(); @@ -626,7 +646,7 @@ int worry_about_NULs; indent_puts( "if ( *yy_cp )" ); indent_up(); - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); } if ( fulltbl ) @@ -644,7 +664,7 @@ int worry_about_NULs; if ( worry_about_NULs && nultrans ) { - /* { for vi */ + indent_puts( "}" ); indent_down(); indent_puts( "else" ); @@ -1132,7 +1152,7 @@ void make_tables() set_indent( 0 ); indent_puts( "struct yy_trans_info" ); indent_up(); - indent_puts( "{" ); /* } for vi */ + indent_puts( "{" ); if ( long_align ) indent_puts( "long yy_verify;" ); @@ -1159,6 +1179,9 @@ void make_tables() else gentabs(); + if ( do_yylineno ) + geneoltbl(); + /* Definitions for backing up. We don't need them if REJECT * is being used because then we use an alternative backin-up * technique instead. @@ -1232,7 +1255,7 @@ void make_tables() } outn( "#define REJECT \\" ); - outn( "{ \\" ); /* } for vi */ + outn( "{ \\" ); outn( "*yy_cp = YY_G(yy_hold_char); /* undo effects of setting up yytext */ \\" ); outn( @@ -1250,7 +1273,7 @@ void make_tables() outn( "++yy_lp; \\" ); outn( "goto find_rule; \\" ); - /* { for vi */ + outn( "}" ); } @@ -1446,9 +1469,8 @@ void make_tables() gen_find_action(); skelout(); /* %% [11.0] - break point in skel */ - if ( do_yylineno ) - { - indent_puts( "if ( yy_act != YY_END_OF_BUFFER )" ); + outn( "#ifdef YY_USE_LINENO" ); + indent_puts( "if ( yy_act != YY_END_OF_BUFFER && yy_rule_can_match_eol[yy_act] )" ); indent_up(); indent_puts( "{" ); indent_puts( "int yyl;" ); @@ -1464,7 +1486,7 @@ void make_tables() indent_down(); indent_puts( "}" ); indent_down(); - } + outn( "#endif" ); skelout(); /* %% [12.0] - break point in skel */ if ( ddebug ) @@ -1623,14 +1645,6 @@ void make_tables() gen_NUL_trans(); skelout(); /* %% [18.0] - break point in skel */ - if ( do_yylineno ) - { /* update yylineno inside of unput() */ - indent_puts( "if ( c == '\\n' )" ); - indent_up(); - indent_puts( "--yylineno;" ); - indent_down(); - } - skelout(); /* %% [19.0] - break point in skel */ /* Update BOL and yylineno inside of input(). */ if ( bol_needed ) @@ -104,6 +104,8 @@ int end_of_buffer_state; char **input_files; int num_input_files; jmp_buf flex_main_jmp_buf; +bool *rule_has_nl, *ccl_has_nl; +int nlch = '\n'; /* Make sure program_name is initialized so we don't crash if writing * out an error message before getting the program name from argv[0]. @@ -227,9 +229,12 @@ void check_options() if ( use_stdout && headerfilename ) flexerror( _( "Can't specify header option if writing to stdout.") ); +#if 0 + /* This makes no sense whatsoever. I'm removing it. */ if ( do_yylineno ) /* This should really be "maintain_backup_tables = true" */ reject_really_used = true; +#endif if ( csize == unspecified ) { @@ -260,9 +265,6 @@ void check_options() flexerror( _( "-Cf/-CF are incompatible with lex-compatibility mode" ) ); - if ( do_yylineno ) - flexerror( - _( "-Cf/-CF and %option yylineno are incompatible" ) ); if ( fulltbl && fullspd ) flexerror( _( "-Cf and -CF are mutually exclusive" ) ); @@ -398,8 +400,6 @@ void check_options() GEN_PREFIX( "set_lloc" ); outn( "#endif" ); - if ( do_yylineno && reentrant) - outn ( "#define YY_USE_LINENO 1"); } if ( do_yylineno && !reentrant ) @@ -414,9 +414,12 @@ void check_options() if ( did_outfilename ) line_directive_out( stdout, 0 ); - /* Dump the user defined preproc directives. */ - if (userdef_buf.elts) - outn( (char*)(userdef_buf.elts) ); + if ( do_yylineno ) + buf_strdefine(&userdef_buf,"YY_USE_LINENO","1"); + + /* Dump the user defined preproc directives. */ + if (userdef_buf.elts) + outn( (char*)(userdef_buf.elts) ); skelout(); /* %% [1.0] */ @@ -1506,7 +1509,7 @@ _( "Variable trailing context rules entail a large performance penalty\n" ) ); _( "REJECT cannot be used with -f or -F" ) ); else if ( do_yylineno ) flexerror( - _( "%option yylineno cannot be used with -f or -F" ) ); + _( "%option yylineno cannot be used with REJECT" ) ); else flexerror( _( "variable trailing context rules cannot be used with -f or -F" ) ); @@ -1585,12 +1588,14 @@ _( "Variable trailing context rules entail a large performance penalty\n" ) ); if ( lex_compat ) outn( "#define YY_FLEX_LEX_COMPAT" ); - if ( do_yylineno && ! C_plus_plus && ! reentrant ) + if ( ! C_plus_plus && ! reentrant ) { + outn( "#ifdef YY_USE_LINENO" ); outn( "extern int yylineno;" ); OUT_BEGIN_CODE(); outn( "int yylineno = 1;" ); OUT_END_CODE(); + outn( "#endif" ); } if ( C_plus_plus ) @@ -1674,6 +1679,7 @@ void set_up_initial_allocations() rule_type = allocate_integer_array( current_max_rules ); rule_linenum = allocate_integer_array( current_max_rules ); rule_useful = allocate_integer_array( current_max_rules ); + rule_has_nl = allocate_bool_array( current_max_rules ); current_max_scs = INITIAL_MAX_SCS; scset = allocate_integer_array( current_max_scs ); @@ -1686,6 +1692,7 @@ void set_up_initial_allocations() cclmap = allocate_integer_array( current_maxccls ); ccllen = allocate_integer_array( current_maxccls ); cclng = allocate_integer_array( current_maxccls ); + ccl_has_nl = allocate_bool_array( current_maxccls ); current_max_ccl_tbl_size = INITIAL_MAX_CCL_TBL_SIZE; ccltbl = allocate_Character_array( current_max_ccl_tbl_size ); @@ -198,8 +198,8 @@ int mach; * context has variable length. */ -void finish_rule( mach, variable_trail_rule, headcnt, trailcnt ) -int mach, variable_trail_rule, headcnt, trailcnt; +void finish_rule( mach, variable_trail_rule, headcnt, trailcnt, pcont_act ) +int mach, variable_trail_rule, headcnt, trailcnt, pcont_act; { char action_text[MAXLINE]; @@ -216,8 +216,19 @@ int mach, variable_trail_rule, headcnt, trailcnt; if ( continued_action ) --rule_linenum[num_rules]; + + /* If the previous rule was continued action, then we inherit the + * previous newline flag, possibly overriding the current one. + */ + if ( pcont_act && rule_has_nl[num_rules-1] ) + rule_has_nl[num_rules] = true; + sprintf( action_text, "case %d:\n", num_rules ); add_action( action_text ); + sprintf( action_text, "/* rule %d can match eol = %s*/\n", + num_rules, rule_has_nl[num_rules]? "true":"false"); + add_action( action_text ); + if ( variable_trail_rule ) { @@ -702,6 +713,8 @@ void new_rule() current_max_rules ); rule_useful = reallocate_integer_array( rule_useful, current_max_rules ); + rule_has_nl = reallocate_bool_array( rule_has_nl, + current_max_rules ); } if ( num_rules > MAX_RULE ) @@ -709,4 +722,5 @@ void new_rule() rule_linenum[num_rules] = linenum; rule_useful[num_rules] = false; + rule_has_nl[num_rules] = false; } @@ -109,12 +109,12 @@ int previous_continued_action; /* whether the previous rule's action was '|' */ /* Expand a POSIX character class expression. */ #define CCL_EXPR(func) \ - { \ + do{ \ int c; \ for ( c = 0; c < csize; ++c ) \ if ( isascii(c) && func(c) ) \ ccladd( currccl, c ); \ - } + }while(0) /* While POSIX defines isblank(), it's not ANSI C. */ #define IS_BLANK(c) ((c) == ' ' || (c) == '\t') @@ -144,7 +144,7 @@ goal : initlex sect1 sect1end sect2 initforrule */ default_rule = num_rules; - finish_rule( def_rule, false, 0, 0 ); + finish_rule( def_rule, false, 0, 0, 0); for ( i = 1; i <= lastsc; ++i ) scset[i] = mkbranch( scset[i], def_rule ); @@ -243,7 +243,7 @@ flexrule : '^' rule { pat = $2; finish_rule( pat, variable_trail_rule, - headcnt, trailcnt ); + headcnt, trailcnt , previous_continued_action); if ( scon_stk_ptr > 0 ) { @@ -279,7 +279,7 @@ flexrule : '^' rule { pat = $1; finish_rule( pat, variable_trail_rule, - headcnt, trailcnt ); + headcnt, trailcnt , previous_continued_action); if ( scon_stk_ptr > 0 ) { @@ -415,6 +415,7 @@ rule : re2 re /* Mark as variable. */ varlength = true; headcnt = 0; + } if ( lex_compat || (varlength && headcnt == 0) ) @@ -734,6 +735,9 @@ singleton : singleton '*' ++rulelen; + if (ccl_has_nl[$1]) + rule_has_nl[num_rules] = true; + $$ = mkstate( -$1 ); } @@ -741,6 +745,9 @@ singleton : singleton '*' { ++rulelen; + if (ccl_has_nl[$1]) + rule_has_nl[num_rules] = true; + $$ = mkstate( -$1 ); } @@ -757,6 +764,9 @@ singleton : singleton '*' if ( caseins && $1 >= 'A' && $1 <= 'Z' ) $1 = clower( $1 ); + if ($1 == nlch) + rule_has_nl[num_rules] = true; + $$ = mkstate( $1 ); } ; @@ -825,23 +835,23 @@ ccl : ccl CHAR '-' CHAR } ; -ccl_expr: CCE_ALNUM { CCL_EXPR(isalnum) } - | CCE_ALPHA { CCL_EXPR(isalpha) } - | CCE_BLANK { CCL_EXPR(IS_BLANK) } - | CCE_CNTRL { CCL_EXPR(iscntrl) } - | CCE_DIGIT { CCL_EXPR(isdigit) } - | CCE_GRAPH { CCL_EXPR(isgraph) } - | CCE_LOWER { CCL_EXPR(islower) } - | CCE_PRINT { CCL_EXPR(isprint) } - | CCE_PUNCT { CCL_EXPR(ispunct) } - | CCE_SPACE { CCL_EXPR(isspace) } +ccl_expr: CCE_ALNUM { CCL_EXPR(isalnum); } + | CCE_ALPHA { CCL_EXPR(isalpha); } + | CCE_BLANK { CCL_EXPR(IS_BLANK); } + | CCE_CNTRL { CCL_EXPR(iscntrl); } + | CCE_DIGIT { CCL_EXPR(isdigit); } + | CCE_GRAPH { CCL_EXPR(isgraph); } + | CCE_LOWER { CCL_EXPR(islower); } + | CCE_PRINT { CCL_EXPR(isprint); } + | CCE_PUNCT { CCL_EXPR(ispunct); } + | CCE_SPACE { CCL_EXPR(isspace); } | CCE_UPPER { if ( caseins ) - CCL_EXPR(islower) + CCL_EXPR(islower); else - CCL_EXPR(isupper) + CCL_EXPR(isupper); } - | CCE_XDIGIT { CCL_EXPR(isxdigit) } + | CCE_XDIGIT { CCL_EXPR(isxdigit); } ; string : string CHAR @@ -849,6 +859,9 @@ string : string CHAR if ( caseins && $2 >= 'A' && $2 <= 'Z' ) $2 = clower( $2 ); + if ( $2 == nlch ) + rule_has_nl[num_rules] = true; + ++rulelen; $$ = link_machines( $1, mkstate( $2 ) ); @@ -279,7 +279,7 @@ LEXOPT [aceknopr] unput ACTION_IFDEF("YY_NO_UNPUT", ! option_sense); verbose printstats = option_sense; warn nowarn = ! option_sense; - yylineno do_yylineno = option_sense; + yylineno do_yylineno = option_sense; ACTION_IFDEF("YY_USE_LINENO", option_sense); yymore yymore_really_used = option_sense; yywrap do_yywrap = option_sense; |