major overhaul for merged skeleton

1 files changed, 600 insertions, 177 deletions

diff --git a/gen.c b/gen.c
index a17f709..52a42a6 100644
--- a/gen.c
+++ b/gen.c
@@ -20,6 +20,87 @@ static char rcsid[] =
 #endif
 
 
+static int indent_level = 0; /* each level is 4 spaces */
+
+#define indent_up() (++indent_level)
+#define indent_down() (--indent_level)
+#define set_indent(indent_val) indent_level = indent_val
+
+
+
+/* indent to the current level */
+
+do_indent()
+
+    {
+    register int i = indent_level * 4;
+
+    while ( i >= 8 )
+	{
+	putchar( '\t' );
+	i -= 8;
+	}
+    
+    while ( i > 0 )
+	{
+	putchar( ' ' );
+	--i;
+	}
+    }
+
+
+/* generate the code to keep backtracking information */
+
+gen_backtracking()
+
+    {
+    if ( reject || num_backtracking == 0 )
+	return;
+
+    if ( fullspd )
+	indent_puts( "if ( yy_current_state[-1].yy_nxt )" );
+    else
+	indent_puts( "if ( yy_accept[yy_current_state] )" );
+
+    indent_up();
+    indent_puts( "{" );
+    indent_puts( "yy_last_accepting_state = yy_current_state;" );
+    indent_puts( "yy_last_accepting_cpos = yy_cp;" );
+    indent_puts( "}" );
+    indent_down();
+    }
+
+
+/* generate the code to perform the backtrack */
+
+gen_bt_action()
+
+    {
+    if ( reject || num_backtracking == 0 )
+	return;
+
+    set_indent( 4 );
+
+    indent_puts( "case 0: /* must backtrack */" );
+    indent_puts( "/* undo the effects of YY_DO_BEFORE_ACTION */" );
+    indent_puts( "*yy_cp = yy_hold_char;" );
+
+    if ( fullspd || fulltbl )
+	indent_puts( "yy_cp = yy_last_accepting_cpos + 1;" );
+    else
+	/* backtracking info for compressed tables is taken \after/
+	 * yy_cp has been incremented for the next state
+	 */
+	indent_puts( "yy_cp = yy_last_accepting_cpos;" );
+
+    indent_puts( "yy_current_state = yy_last_accepting_state;" );
+    indent_puts( "continue; /* go to \"YY_DO_BEFORE_ACTION\" */" );
+    putchar( '\n' );
+
+    set_indent( 0 );
+    }
+
+
 /* genctbl - generates full speed compressed transition table
  *
  * synopsis
@@ -30,6 +111,7 @@ genctbl()
 
     {
     register int i;
+    int end_of_buffer_action = num_rules + 1;
 
     /* table of verify for transition and offset to next state */
     printf( "static struct yy_trans_info yy_transition[%d] =\n",
@@ -55,7 +137,7 @@ genctbl()
      */
 
     base[lastdfa + 1] = tblend + 2;
-    nxt[tblend + 1] = END_OF_BUFFER_ACTION;
+    nxt[tblend + 1] = end_of_buffer_action;
     chk[tblend + 1] = numecs + 1;
     chk[tblend + 2] = 1; /* anything but EOB */
     nxt[tblend + 2] = 0; /* so that "make test" won't show arb. differences */
@@ -67,7 +149,7 @@ genctbl()
 
 	chk[base[i]] = EOB_POSITION;
 	chk[base[i] - 1] = ACTION_POSITION;
-	nxt[base[i] - 1] = anum ? anum : accnum + 1;	/* action number */
+	nxt[base[i] - 1] = anum;	/* action number */
 	}
 
     dataline = 0;
@@ -97,7 +179,7 @@ genctbl()
     printf( "\n" );
 
     /* table of pointers to start states */
-    printf( "static struct yy_trans_info *yy_state_ptr[%d] =\n",
+    printf( "static struct yy_trans_info *yy_start_state_list[%d] =\n",
 	lastsc * 2 + 1 );
     printf( "    {\n" );
 
@@ -111,6 +193,170 @@ genctbl()
     }
 
 
+/* generate equivalence-class tables */
+
+genecs()
+
+    {
+    register int i, j;
+    static char C_char_decl[] = "static char %s[%d] =\n    {   0,\n";
+    int numrows;
+    char clower();
+
+    printf( C_char_decl, ECARRAY, CSIZE + 1 );
+
+    for ( i = 1; i <= CSIZE; ++i )
+	{
+	if ( caseins && (i >= 'A') && (i <= 'Z') )
+	    ecgroup[i] = ecgroup[clower( i )];
+
+	ecgroup[i] = abs( ecgroup[i] );
+	mkdata( ecgroup[i] );
+	}
+
+    dataend();
+
+    if ( trace )
+	{
+	fputs( "\n\nEquivalence Classes:\n\n", stderr );
+
+	numrows = (CSIZE + 1) / 8;
+
+	for ( j = 1; j <= numrows; ++j )
+	    {
+	    for ( i = j; i <= CSIZE; i = i + numrows )
+		{
+		char *readable_form();
+
+		fprintf( stderr, "%4s = %-2d",
+			 readable_form( i ), ecgroup[i] );
+
+		putc( ' ', stderr );
+		}
+
+	    putc( '\n', stderr );
+	    }
+	}
+    }
+
+
+/* generate the code to find the action number */
+
+gen_find_action()
+
+    {
+    if ( fullspd )
+	indent_puts( "yy_act = yy_current_state[-1].yy_nxt;" );
+
+    else if ( fulltbl )
+	indent_puts( "yy_act = yy_accept[yy_current_state];" );
+
+    else if ( reject )
+	{
+	indent_puts( "yy_current_state = *--yy_state_ptr;" );
+	indent_puts( "yy_lp = yy_accept[yy_current_state];" );
+
+	puts( "find_rule: /* we branch to this label when backtracking */" );
+
+	indent_puts( "for ( ; ; ) /* until we find what rule we matched */" );
+
+	indent_up();
+
+	indent_puts( "{" );
+
+	indent_puts( "if ( yy_lp && yy_lp < yy_accept[yy_current_state + 1] )" );
+	indent_up();
+	indent_puts( "{" );
+	indent_puts( "yy_act = yy_acclist[yy_lp];" );
+
+	if ( variable_trailing_context_rules )
+	    {
+	    indent_puts( "if ( yy_act & YY_TRAILING_HEAD_MASK ||" );
+	    indent_puts( "     yy_looking_for_trail_begin )" );
+	    indent_up();
+	    indent_puts( "{" );
+
+	    indent_puts( "if ( yy_act == yy_looking_for_trail_begin )" );
+	    indent_up();
+	    indent_puts( "{" );
+	    indent_puts( "yy_looking_for_trail_begin = 0;" );
+	    indent_puts( "yy_act &= ~YY_TRAILING_HEAD_MASK;" );
+	    indent_puts( "break;" );
+	    indent_puts( "}" );
+	    indent_down();
+
+	    indent_puts( "}" );
+	    indent_down();
+
+	    indent_puts( "else if ( yy_act & YY_TRAILING_MASK )" );
+	    indent_up();
+	    indent_puts( "{" );
+	    indent_puts(
+		"yy_looking_for_trail_begin = yy_act & ~YY_TRAILING_MASK;" );
+	    indent_puts(
+		"yy_looking_for_trail_begin |= YY_TRAILING_HEAD_MASK;" );
+
+	    if ( real_reject )
+		{
+		/* remember matched text in case we back up due to REJECT */
+		indent_puts( "yy_full_match = yy_cp;" );
+		indent_puts( "yy_full_state = yy_state_ptr;" );
+		indent_puts( "yy_full_lp = yy_lp;" );
+		}
+
+	    indent_puts( "}" );
+	    indent_down();
+
+	    indent_puts( "else" );
+	    indent_up();
+	    indent_puts( "{" );
+	    indent_puts( "yy_full_match = yy_cp;" );
+	    indent_puts( "yy_full_state = yy_state_ptr;" );
+	    indent_puts( "yy_full_lp = yy_lp;" );
+	    indent_puts( "break;" );
+	    indent_puts( "}" );
+	    indent_down();
+
+	    indent_puts( "++yy_lp;" );
+	    indent_puts( "goto find_rule;" );
+	    }
+
+	else
+	    {
+	    /* remember matched text in case we back up due to trailing context
+	     * plus REJECT
+	     */
+	    indent_up();
+	    indent_puts( "{" );
+	    indent_puts( "yy_full_match = yy_cp;" );
+	    indent_puts( "break;" );
+	    indent_puts( "}" );
+	    indent_down();
+	    }
+
+	indent_puts( "}" );
+	indent_down();
+
+	indent_puts( "--yy_cp;" );
+
+	/* we could consolidate the following two lines with those at
+	 * the beginning, but at the cost of complaints that we're
+	 * branching inside a loop
+	 */
+	indent_puts( "yy_current_state = *--yy_state_ptr;" );
+	indent_puts( "yy_lp = yy_accept[yy_current_state];" );
+
+	indent_puts( "}" );
+
+	indent_down();
+	}
+
+    else
+	/* compressed */
+	indent_puts( "yy_act = yy_accept[yy_current_state];" );
+    }
+
+
 /* genftbl - generates full transition table
  *
  * synopsis
@@ -121,28 +367,23 @@ genftbl()
 
     {
     register int i;
+    int end_of_buffer_action = num_rules + 1;
 
     /* *everything* is done in terms of arrays starting at 1, so provide
      * a null entry for the zero element of all C arrays
      */
-    static char C_short_decl[] = "static short int %c[%d] =\n    {   0,\n";
-    static char C_char_decl[] = "static char %c[%d] =\n    {   0,\n";
+    static char C_short_decl[] = "static short int %s[%d] =\n    {   0,\n";
 
-#ifdef UNSIGNED_CHAR
     printf( C_short_decl, ALIST, lastdfa + 1 );
-#else
-    printf( accnum > 127 ? C_short_decl : C_char_decl, ALIST, lastdfa + 1 );
-#endif
+
+
+    dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action;
 
     for ( i = 1; i <= lastdfa; ++i )
 	{
 	register int anum = dfaacc[i].dfaacc_state;
 
-	if ( i == end_of_buffer_state )
-	    mkdata( END_OF_BUFFER_ACTION );
-
-	else
-	    mkdata( anum ? anum : accnum + 1 );
+	mkdata( anum );
 
 	if ( trace && anum )
 	    fprintf( stderr, "state # %d accepts: [%d]\n", i, anum );
@@ -159,6 +400,205 @@ genftbl()
     }
 
 
+/* generate the code to find the next compressed-table state */
+
+gen_next_compressed_state()
+
+    {
+    char *char_map = useecs ? "yy_ec[*yy_cp]" : "*yy_cp";
+
+    indent_put2s( "register char yy_c = %s;", char_map );
+
+    /* save the backtracking info \before/ computing the next state
+     * because we always compute one more state than needed - we
+     * always proceed until we reach a jam state
+     */
+    gen_backtracking();
+
+    indent_puts(
+    "while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )" );
+    indent_up();
+    indent_puts( "{" );
+    indent_puts( "yy_current_state = yy_def[yy_current_state];" );
+
+    if ( usemecs )
+	{
+	/* we've arrange it so that templates are never chained
+	 * to one another.  This means we can afford make a
+	 * very simple test to see if we need to convert to
+	 * yy_c's meta-equivalence class without worrying
+	 * about erroneously looking up the meta-equivalence
+	 * class twice
+	 */
+	do_indent();
+	/* lastdfa + 2 is the beginning of the templates */
+	printf( "if ( yy_current_state >= %d )\n", lastdfa + 2 );
+
+	indent_up();
+	indent_puts( "yy_c = yy_meta[yy_c];" );
+	indent_down();
+	}
+
+    indent_puts( "}" );
+    indent_down();
+
+    indent_puts(
+	"yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c];" );
+    }
+
+
+/* generate the code to find the next match */
+
+gen_next_match()
+
+    { /* NOTE - changes in here should be reflected in get_next_state() */
+    char *char_map = useecs ? "yy_ec[*yy_cp]" : "*yy_cp";
+    char *char_map_2 = useecs ? "yy_ec[*++yy_cp]" : "*++yy_cp";
+    
+    if ( fulltbl )
+	{
+	indent_put2s(
+	    "while ( (yy_current_state = yy_nxt[yy_current_state][%s]) > 0 )",
+		char_map );
+
+	indent_up();
+
+	if ( num_backtracking > 0 )
+	    {
+	    indent_puts( "{" );
+	    gen_backtracking();
+	    putchar( '\n' );
+	    }
+
+	indent_puts( "++yy_cp;" );
+
+	if ( num_backtracking > 0 )
+	    indent_puts( "}" );
+
+	indent_down();
+
+	putchar( '\n' );
+	indent_puts( "yy_current_state = -yy_current_state;" );
+	}
+
+    else if ( fullspd )
+	{
+	indent_puts( "{" );
+	indent_puts( "register struct yy_trans_info *yy_trans_info;\n" );
+	indent_puts( "register char yy_c;\n" );
+	indent_put2s( "for ( yy_c = %s;", char_map );
+	indent_puts(
+	"      (yy_trans_info = &yy_current_state[yy_c])->yy_verify == yy_c;" );
+	indent_put2s( "      yy_c = %s )", char_map_2 );
+
+	indent_up();
+
+	if ( num_backtracking > 0 )
+	    indent_puts( "{" );
+
+	indent_puts( "yy_current_state += yy_trans_info->yy_nxt;" );
+
+	if ( num_backtracking > 0 )
+	    {
+	    putchar( '\n' );
+	    gen_backtracking();
+	    indent_puts( "}" );
+	    }
+
+	indent_down();
+	indent_puts( "}" );
+	}
+
+    else
+	{ /* compressed */
+	indent_puts( "do" );
+
+	indent_up();
+	indent_puts( "{" );
+
+	gen_next_state();
+
+	indent_puts( "++yy_cp;" );
+
+	indent_puts( "}" );
+	indent_down();
+
+	do_indent();
+
+	if ( interactive )
+	    printf( "while ( yy_base[yy_current_state] != %d );\n", jambase );
+	else
+	    printf( "while ( yy_current_state != %d );\n", jamstate );
+
+	if ( ! reject )
+	    {
+	    /* do the guaranteed-needed backtrack to figure out the match */
+	    indent_puts( "yy_cp = yy_last_accepting_cpos;" );
+	    indent_puts( "yy_current_state = yy_last_accepting_state;" );
+	    }
+	}
+    }
+
+
+/* generate the code to find the next state */
+
+gen_next_state()
+
+    { /* NOTE - changes in here should be reflected in get_next_match() */
+    char *char_map = useecs ? "yy_ec[*yy_cp]" : "*yy_cp";
+    
+    if ( fulltbl )
+	{
+	indent_put2s( "yy_current_state = yy_nxt[yy_current_state][%s];", 
+		char_map );
+	gen_backtracking();
+	}
+
+    else if ( fullspd )
+	{
+	indent_put2s( "yy_current_state += yy_current_state[%s].yy_nxt;",
+		char_map );
+	gen_backtracking();
+	}
+
+    else
+	{
+	gen_next_compressed_state();
+
+	if ( reject )
+	    indent_puts( "*yy_state_ptr++ = yy_current_state;" );
+	}
+    }
+
+
+/* generate the code to find the start state */
+
+gen_start_state()
+
+    {
+    if ( fullspd )
+	indent_put2s( "yy_current_state = yy_start_state_list[yy_start%s];",
+		bol_needed ? " + (yy_bp[-1] == '\\n' ? 1 : 0)" : "" );
+
+    else
+	{
+	indent_puts( "yy_current_state = yy_start;" );
+
+	if ( bol_needed )
+	    {
+	    indent_puts( "if ( yy_bp[-1] == '\\n' )" );
+	    indent_up();
+	    indent_puts( "++yy_current_state;" );
+	    indent_down();
+	    }
+
+	if ( reject )
+	    /* set up for storing up states */
+	    indent_puts( "yy_state_ptr = yy_state_buf;" );
+	}
+    }
+
+
 /* gentabs - generate data statements for the transition tables
  *
  * synopsis
@@ -169,33 +609,42 @@ gentabs()
 
     {
     int i, j, k, *accset, nacc, *acc_array, total_states;
+    int end_of_buffer_action = num_rules + 1;
 
     /* *everything* is done in terms of arrays starting at 1, so provide
      * a null entry for the zero element of all C arrays
      */
-    static char C_long_decl[] = "static long int %c[%d] =\n    {   0,\n";
-    static char C_short_decl[] = "static short int %c[%d] =\n    {   0,\n";
-    static char C_char_decl[] = "static char %c[%d] =\n    {   0,\n";
+    static char C_long_decl[] = "static long int %s[%d] =\n    {   0,\n";
+    static char C_short_decl[] = "static short int %s[%d] =\n    {   0,\n";
+    static char C_char_decl[] = "static char %s[%d] =\n    {   0,\n";
 
     acc_array = allocate_integer_array( current_max_dfas );
     nummt = 0;
 
-    printf( "#define YY_JAM %d\n", jamstate );
-    printf( "#define YY_JAM_BASE %d\n", jambase );
-
-    if ( usemecs )
-	printf( "#define YY_TEMPLATE %d\n", lastdfa + 2 );
+    /* the compressed table format jams by entering the "jam state",
+     * losing information about the previous state in the process.
+     * In order to recover the previous state, we effectively need
+     * to keep backtracking information.
+     */
+    ++num_backtracking;
 
     if ( reject )
 	{
 	/* write out accepting list and pointer list
+	 *
 	 * first we generate the ACCEPT array.  In the process, we compute
 	 * the indices that will go into the ALIST array, and save the
 	 * indices in the dfaacc array
 	 */
+	int EOB_accepting_list[2];
 
-	printf( accnum > 127 ? C_short_decl : C_char_decl,
-		ACCEPT, max( numas, 1 ) + 1 );
+	printf( C_short_decl, ACCEPT, max( numas, 1 ) + 1 );
+	
+	/* set up accepting structures for the End Of Buffer state */
+	EOB_accepting_list[0] = 0;
+	EOB_accepting_list[1] = end_of_buffer_action;
+	accsiz[end_of_buffer_state] = 1;
+	dfaacc[end_of_buffer_state].dfaacc_set = EOB_accepting_list;
 
 	j = 1;	/* index into ACCEPT array */
 
@@ -213,8 +662,20 @@ gentabs()
 
 		for ( k = 1; k <= nacc; ++k )
 		    {
+		    int accnum = accset[k];
+
 		    ++j;
-		    mkdata( accset[k] );
+
+		    if ( variable_trailing_context_rules && accnum > 0 &&
+			 rule_type[accnum] == RULE_VARIABLE )
+			{
+			/* special hack to flag accepting number as part
+			 * of trailing context rule
+			 */
+			accnum |= YY_TRAILING_MASK;
+			}
+
+		    mkdata( accnum );
 
 		    if ( trace )
 			{
@@ -237,6 +698,8 @@ gentabs()
 
     else
 	{
+	dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action;
+
 	for ( i = 1; i <= lastdfa; ++i )
 	    acc_array[i] = dfaacc[i].dfaacc_state;
 
@@ -262,12 +725,7 @@ gentabs()
 	 */
 	++k;
 
-#ifdef UNSIGNED_CHAR
     printf( C_short_decl, ALIST, k );
-#else
-    printf( ((reject && numas > 126) || accnum > 127) ?
-	    C_short_decl : C_char_decl, ALIST, k );
-#endif
 
     for ( i = 1; i <= lastdfa; ++i )
 	{
@@ -381,145 +839,30 @@ gentabs()
     }
 
 
-/* generate equivalence-class tables */
+/* write out a formatted string (with a secondary string argument) at the
+ * current indentation level, adding a final newline
+ */
 
-genecs()
+indent_put2s( fmt, arg )
+char fmt[], arg[];
 
     {
-    register int i, j;
-    static char C_char_decl[] = "static char %c[%d] =\n    {   0,\n";
-    int numrows;
-    char clower();
-
-    printf( C_char_decl, ECARRAY, CSIZE + 1 );
-
-    for ( i = 1; i <= CSIZE; ++i )
-	{
-	if ( caseins && (i >= 'A') && (i <= 'Z') )
-	    ecgroup[i] = ecgroup[clower( i )];
-
-	ecgroup[i] = abs( ecgroup[i] );
-	mkdata( ecgroup[i] );
-	}
-
-    dataend();
-
-    if ( trace )
-	{
-	fputs( "\n\nEquivalence Classes:\n\n", stderr );
-
-	numrows = (CSIZE + 1) / 8;
-
-	for ( j = 1; j <= numrows; ++j )
-	    {
-	    for ( i = j; i <= CSIZE; i = i + numrows )
-		{
-		char *readable_form();
-
-		fprintf( stderr, "%4s = %-2d",
-			 readable_form( i ), ecgroup[i] );
-
-		putc( ' ', stderr );
-		}
-
-	    putc( '\n', stderr );
-	    }
-	}
+    do_indent();
+    printf( fmt, arg );
+    putchar( '\n' );
     }
 
 
-/* generate the code to find the next state */
+/* write out a string at the current indentation level, adding a final
+ * newline
+ */
 
-gen_next_state()
+indent_puts( str )
+char str[];
 
     {
-    char *char_map = useecs ? "e[*yy_cp]" : "*yy_cp";
-    
-    if ( fulltbl )
-	{
-	printf( "while ( (yy_current_state = n[yy_current_state][%s]) > 0 )\n",
-		char_map );
-
-	puts( "\t{" );
-
-	if ( num_backtracking > 0 )
-	    {
-	    puts( "\tif ( l[yy_current_state] )" );
-	    puts( "\t\t{" );
-	    puts( "\t\tyy_last_accepting_state = yy_current_state;" );
-	    puts( "\t\tyy_last_accepting_cpos = yy_cp;" );
-	    puts( "\t\t}" );
-	    }
-
-	puts( "yy_cp++;" );
-	puts( "\t}" );
-
-	puts( "yy_current_state = -yy_current_state;" );
-	}
-
-    else if ( fullspd )
-	{
-	puts( "{" );
-	puts( "register char yy_c;" );
-	printf( "\nfor ( yy_c = %s;\n", char_map );
-	puts( "      (yy_trans_info = &yy_current_state[yy_c])->v == yy_c;" );
-	printf( "      yy_c = %s )\n", char_map );
-
-	puts( "\t{" );
-
-	puts( "\tyy_current_state += yy_trans_info->n;" );
-
-	if ( num_backtracking > 0 )
-	    {
-	    puts( "\tif ( yy_current_state[-1].n )" );
-	    puts( "\t\t{" );
-	    puts( "yy_last_accepting_state = yy_current_state;" );
-	    puts( "yy_last_accepting_cpos = yy_c_buf_p;" );
-	    puts( "\t\t}" );
-	    }
-
-	puts( "\t}" );
-	puts( "}" );
-	}
-
-    else
-	{ /* compressed */
-	puts( "do" );
-
-	puts( "\t{" );
-	printf( "\tregister char yy_c = %s;\n", char_map );
-	puts(
-	"\twhile ( c[b[yy_current_state] + yy_sym] != yy_current_state )" );
-	puts( "\t\t{" );
-	puts( "yy_current_state = d[yy_current_state];" );
-
-	if ( usemecs )
-	    {
-	    /* we've arrange it so that templates are never chained
-             * to one another.  This means we can afford make a
-             * very simple test to see if we need to convert to
-             * yy_c's meta-equivalence class without worrying
-             * about erroneously looking up the meta-equivalence
-             * class twice
-             */
-	    puts( "\t\tif ( yy_current_state >= YY_TEMPLATE )" );
-	    puts( "\t\t\tyy_c = m[yy_c];" );
-	    }
-
-	puts( "\t\t}" );
-
-	puts( "\tyy_current_state = n[b[yy_current_state] + yy_c];" );
-
-	if ( reject )
-	    puts( "\t*yy_state_ptr++ = yy_current_state;" );
-
-	puts( "\t}" );
-
-	if ( interactive )
-	    puts( "while ( b[yy_current_state] != YY_JAM_BASE );" );
-	else
-	    puts( "while ( yy_current_state != YY_JAM );" );
-	}
+    do_indent();
+    puts( str );
     }
 
 
@@ -534,34 +877,94 @@ gen_next_state()
 make_tables()
 
     {
+    printf( "#define YY_END_OF_BUFFER %d\n", num_rules + 1 );
+
     if ( fullspd )
-	{ /* need to define YY_TRANS_OFFSET_TYPE as a size large
+	{ /* need to define the transet type as a size large
 	   * enough to hold the biggest offset
 	   */
 	int total_table_size = tblend + numecs + 1;
-
-	printf( "#define YY_TRANS_OFFSET_TYPE %s\n",
-		total_table_size > MAX_SHORT ? "long" : "short" );
+	char *trans_offset_type =
+	    total_table_size > MAX_SHORT ? "long" : "short";
+
+	set_indent( 0 );
+	indent_puts( "struct yy_trans_info" );
+	indent_up();
+        indent_puts( "{" );
+        indent_puts( "short yy_verify;" );
+
+        /* in cases where its sister yy_verify *is* a "yes, there is a
+	 * transition", yy_nxt is the offset (in records) to the next state.
+	 * In most cases where there is no transition, the value of yy_nxt
+	 * is irrelevant.  If yy_nxt is the -1th  record of a state, though,
+	 * then yy_nxt is the action number for that state
+         */
+
+        indent_put2s( "%s yy_nxt;", trans_offset_type );
+        indent_puts( "};" );
+	indent_down();
+
+	indent_puts( "typedef struct yy_trans_info *yy_state_type;" );
 	}
-
-    skelout();
     
-    if ( fullspd || fulltbl )
+    else
+	indent_puts( "typedef int yy_state_type;" );
+
+    if ( fullspd )
+	genctbl();
+
+    else if ( fulltbl )
+	genftbl();
+
+    else
+	gentabs();
+
+    if ( reject )
 	{
-	if ( num_backtracking > 0 )
+	/* declare state buffer variables */
+	puts( "yy_trans_info yy_state_buf[YY_BUF_SIZE + 2], *yy_state_ptr;" );
+	puts( "char *yy_full_match;" );
+	puts( "int yy_lp;" );
+
+	if ( variable_trailing_context_rules )
 	    {
-	    printf( "#define FLEX_USES_BACKTRACKING\n" );
-	    printf( "#define YY_BACK_TRACK %d\n", accnum + 1 );
+	    puts( "int yy_looking_for_trail_begin = 0;" );
+	    puts( "int yy_full_lp;" );
+	    puts( "int *yy_full_state;" );
+	    printf( "#define YY_TRAILING_MASK 0x%x\n", YY_TRAILING_MASK );
+	    printf( "#define YY_TRAILING_HEAD_MASK 0x%x\n",
+		    YY_TRAILING_HEAD_MASK );
 	    }
 
-	if ( fullspd )
-	    genctbl();
-	else
-	    genftbl();
-	}
+	puts( "#define REJECT \\" );
+        puts( "{ \\" );
+        puts(
+	"*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ \\" );
+        puts(
+	    "yy_cp = yy_full_match; /* restore poss. backed-over text */ \\" );
 
+	if ( variable_trailing_context_rules )
+	    {
+	    puts( "yy_lp = yy_full_lp; /* restore orig. accepting pos. */ \\" );
+	    puts(
+		"yy_state_ptr = yy_full_state; /* restore orig. state */ \\" );
+	    puts(
+	    "yy_current_state = *yy_state_ptr; /* restore curr. state */ \\" );
+	    }
+
+        puts( "++yy_lp; \\" );
+        puts( "goto find_rule; \\" );
+        puts( "}" );
+	}
+    
     else
-	gentabs();
+	{
+	puts( "/* the intent behind this definition is that it'll catch" );
+	puts( " * any uses of REJECT which flex missed" );
+	puts( " */" );
+	puts( "#define REJECT reject_used_but_not_detected" );
+	}
+
 
     skelout();
 
@@ -573,12 +976,32 @@ make_tables()
 
     skelout();
 
+    set_indent( 2 );
+
+    gen_start_state();
+    gen_next_match();
+
+    skelout();
+    set_indent( 3 );
+    gen_find_action();
+
     /* copy actions from action_file to output file */
+    skelout();
+    indent_up();
+    gen_bt_action();
     action_out();
 
+    /* generate code for yy_get_previous_state() */
+    set_indent( 1 );
     skelout();
 
-    /* generate code to find next state */
+    if ( bol_needed )
+	indent_puts( "register char *yy_bp = yytext;\n" );
+
+    gen_start_state();
+
+    set_indent( 2 );
+    skelout();
     gen_next_state();
 
     skelout();