diff options
| -rw-r--r-- | TODO | 18 | ||||
| -rw-r--r-- | buf.c | 24 | ||||
| -rw-r--r-- | devel/dump-tables.pl | 94 | ||||
| -rw-r--r-- | dfa.c | 30 | ||||
| -rw-r--r-- | flex.skl | 210 | ||||
| -rw-r--r-- | flexdef.h | 4 | ||||
| -rw-r--r-- | gen.c | 268 | ||||
| -rw-r--r-- | main.c | 30 | ||||
| -rw-r--r-- | misc.c | 31 | ||||
| -rw-r--r-- | options.c | 2 | ||||
| -rw-r--r-- | options.h | 1 | ||||
| -rw-r--r-- | scan.l | 7 | ||||
| -rw-r--r-- | tables.c | 3 | ||||
| -rw-r--r-- | tables.h | 5 | ||||
| -rw-r--r-- | tables_shared.h | 6 | ||||
| -rw-r--r-- | tests/test-table-opts/.cvsignore | 2 | ||||
| -rw-r--r-- | tests/test-table-opts/Makefile.am | 28 | ||||
| -rw-r--r-- | tests/test-table-opts/scanner.l | 38 |
18 files changed, 525 insertions, 276 deletions
@@ -52,6 +52,24 @@ ** revisit the C++ API. We get requests to make it more complete. +* Tables API + +** Fix deserialization of --fast tables + +** Handle reject-triggered tables + +** Lookup table by name + +** rename _fwrite t _write since writer was created + +** create user API for tables deserialization + +** document API and --tables-* options + +** create test for multiple tables + +** verify that new macros/functions work with %option header-file + * build system ** use bootstrapper @@ -36,15 +36,27 @@ /* global buffers. */ struct Buf userdef_buf; /* for user #definitions triggered by cmd-line. */ struct Buf defs_buf; /* for #define's autogenerated. */ +struct Buf yydmap_buf; /* string buffer to hold yydmap elements */ -/* functions for growable buffer. */ +/* Append a "%s" formatted string to a string buffer */ +struct Buf *buf_prints (struct Buf *buf, const char *fmt, const char *s) +{ + char *t; + + t = flex_alloc (strlen (fmt) + strlen (s) + 1); + sprintf (t, fmt, s); + buf = buf_strappend (buf, t); + flex_free (t); + return buf; +} + /* Appends n characters in str to buf. */ struct Buf *buf_strnappend (buf, str, n) struct Buf *buf; const char *str; - int n; + int n; { buf_append (buf, str, n + 1); @@ -78,9 +90,9 @@ struct Buf *buf_strdefine (buf, str, def) } /* create buf with 0 elements, each of size elem_size. */ -void buf_init (buf, elem_size) +void buf_init (buf, elem_size) struct Buf *buf; - size_t elem_size; + size_t elem_size; { buf->elts = (void *) 0; buf->nelts = 0; @@ -89,7 +101,7 @@ void buf_init (buf, elem_size) } /* frees memory */ -void buf_destroy (buf) +void buf_destroy (buf) struct Buf *buf; { if (buf && buf->elts) @@ -107,7 +119,7 @@ void buf_destroy (buf) struct Buf *buf_append (buf, ptr, n_elem) struct Buf *buf; const void *ptr; - int n_elem; + int n_elem; { int n_alloc = 0; diff --git a/devel/dump-tables.pl b/devel/dump-tables.pl index 31919f6..1515e18 100644 --- a/devel/dump-tables.pl +++ b/devel/dump-tables.pl @@ -1,95 +1,21 @@ #! /usr/bin/perl -w # vim:set ft=perl ai si et ts=4 sts=4 sw=4 tw=0: -# USAGE: dump-tables.pl FILE +# USAGE: dump-tables.pl FILE [max-data] use strict; -@ARGV == 1 || die "\nUSAGE: dump-tables.pl FILE\n"; -open FD, $ARGV[0] or die "$ARGV[0]: $!"; - -my ($magic,$hsize) = (read32(),read32()); -my $rest = readn($hsize - 8); -my ($ssize,$flags,$version,$name) = unpack 'Nna*', $rest; -($version,$name) = split /\0/, $version; +my $dir = $0; +$dir =~ s:/[^/]+$::; +push @INC, $dir; +require "tables.pl"; -printf "th_magic: %08X\n", $magic; -print "th_hsize: $hsize\n"; -print "th_ssize: $ssize\n"; -print "th_flags: $flags\n"; -print "th_version: $version\n"; -print "th_name: $name\n"; +@ARGV == 1 || @ARGV == 2 || die "\nUSAGE: dump-tables.pl FILE [max-data]\n"; +open FD, $ARGV[0] or die "$ARGV[0]: $!"; -my $bytes = $hsize; +dump_hdr(read_hdr()); while(!eof FD){ - dump_table(); -} - - -sub readn { my $s; read FD, $s, shift; return $s } -sub read32 { my $n; read FD, $n, 4; return unpack 'i', pack 'i', unpack 'N',$n } -sub read16 { my $n; read FD, $n, 2; return unpack 's', pack 's', unpack 'n',$n } -sub read8 { my $n; read FD, $n, 1; return unpack 'c',$n } - -sub dump_table { - my ($id,$flags,$hi,$lo) = (read16(),read16(),read32(),read32()); - my $max = 40; - print "td_id: $id ($::TID{$id})\n". - "td_flags: $flags (@{[TFLAGS($flags)]})\n". - "td_hilen: $hi\n". - "td_lolen: $lo\n"; - - my $read = $::TFLAGS{$flags}->[1]; - - my $tot = $lo * ($hi?$hi:1); - $tot *=2 if $::TID{$id}eq 'YYT_ID_TRANSITION'; - - my @d; - for(1..$tot){ - my $v = $read->(); - if($max-- > 0){ - push @d, $v; - } - } - print "td_data: "; - print join ', ', @d; - my $pad = pad64(tell FD); - print "\npadding: $pad\n"; - while($pad-- > 0){ read8() } - return tell FD; -} - -sub TFLAGS { - my @s; - my $f = shift; - foreach(keys %::TFLAGS){ - if ($f & $_){ - push @s, $::TFLAGS{$_}->[0] - } - } - return join '|', @s; -} - -sub pad64{ return ((8-((shift)%8))%8) } - -BEGIN { - %::TID = ( - 0x01 => 'YYT_ID_ACCEPT' , - 0x02 => 'YYT_ID_BASE' , - 0x03 => 'YYT_ID_CHK' , - 0x04 => 'YYT_ID_DEF' , - 0x05 => 'YYT_ID_EC' , - 0x06 => 'YYT_ID_META' , - 0x07 => 'YYT_ID_NUL_TRANS' , - 0x08 => 'YYT_ID_NXT' , - 0x09 => 'YYT_ID_RULE_CAN_MATCH_EOL' , - 0x0A => 'YYT_ID_START_STATE_LIST' , - 0x0B => 'YYT_ID_TRANSITION'); - - %::TFLAGS = ( - 0x01 => ['YYT_DATA8',\&read8] , - 0x02 => ['YYT_DATA16',\&read16] , - 0x04 => ['YYT_DATA32',\&read32] , - 0x08 => ['YYT_PTRANS',sub{}]); + dump_table(read_table(),$ARGV[1]); } +close FD; 1 __END__ @@ -534,14 +534,27 @@ void ntod () sizeof (int32_t)); yynxt_curr = 0; + buf_prints (&yydmap_buf, + "\t{YYT_ID_NXT, (void**)&yy_nxt, sizeof(%s)},\n", + long_align ? "int32_t" : "int16_t"); + /* Unless -Ca, declare it "short" because it's a real * long-shot that that won't be large enough. */ - out_str_dec ("static yyconst %s yy_nxt[][%d] =\n {\n", - long_align ? "long" : "short", - num_full_table_rows); + if (gentables) + out_str_dec + ("static yyconst %s yy_nxt[][%d] =\n {\n", + long_align ? "int32_t" : "int16_t", + num_full_table_rows); + else { + out_dec ("#undef YY_NXT_LOLEN\n#define YY_NXT_LOLEN (%d)\n", num_full_table_rows); + out_str ("static yyconst %s *yy_nxt =0;\n", + long_align ? "int32_t" : "int16_t"); + } - outn (" {"); + + if (gentables) + outn (" {"); /* Generate 0 entries for state #0. */ for (i = 0; i < num_full_table_rows; ++i) { @@ -550,7 +563,8 @@ void ntod () } dataflush (); - outn (" },\n"); + if (gentables) + outn (" },\n"); } /* Create the first states. */ @@ -720,7 +734,8 @@ void ntod () sizeof (int32_t)); - outn (" {"); + if (gentables) + outn (" {"); /* Supply array's 0-element. */ if (ds == end_of_buffer_state) { @@ -744,7 +759,8 @@ void ntod () } dataflush (); - outn (" },\n"); + if (gentables) + outn (" },\n"); } else if (fullspd) @@ -32,8 +32,8 @@ #include <stdio.h> #include <errno.h> #include <stdlib.h> -%tables-serialization-code-begin #include <stdint.h> +%tables-serialization-code-begin #include <netinet/in.h> %tables-serialization-code-end /* end standard C headers. */ @@ -762,7 +762,6 @@ int yytables_load YY_PARAMS ((FILE * fp YY_PROTO_LAST_ARG)); * this scanner. This is the bridge between our "generic" deserialization code * and the specifics of this scanner. */ - struct yytbl_dmap { enum yytbl_id dm_id;/**< table identifier */ void **dm_arr; /**< address of pointer to store the deserialized table. */ @@ -776,6 +775,11 @@ static struct yytbl_dmap yydmap[] = {0,0,0} }; +/** A tables-reader object to maintain some state in the read. */ +struct yytbl_reader { + FILE * fp; /**< input stream */ + uint32_t bread; /**< bytes read since beginning of current tableset */ +}; %tables-serialization-code-end structures and prototypes @@ -2071,61 +2075,76 @@ void yyfree YYFARGS1( void *,ptr) %tables-serialization-code-begin definitions m4_include(`tables_shared.c') -static int yytbl_read8 (void *v, FILE * fp) +static int yytbl_read8 (void *v, struct yytbl_reader * rd) { - if (fread (v, sizeof (uint8_t), 1, fp) != 1) + errno = 0; + if (fread (v, sizeof (uint8_t), 1, rd->fp) != 1){ + errno = EIO; return -1; + } + rd->bread += sizeof(uint8_t); return 0; } -static int yytbl_read16 (void *v, FILE * fp) +static int yytbl_read16 (void *v, struct yytbl_reader * rd) { - if (fread (v, sizeof (uint16_t), 1, fp) != 1) + errno = 0; + if (fread (v, sizeof (uint16_t), 1, rd->fp) != 1){ + errno = EIO; return -1; + } *((uint16_t *) v) = ntohs (*((uint16_t *) v)); + rd->bread += sizeof(uint16_t); return 0; } -static int yytbl_read32 (void *v, FILE * fp) +static int yytbl_read32 (void *v, struct yytbl_reader * rd) { - if (fread (v, sizeof (uint32_t), 1, fp) != 1) + errno = 0; + if (fread (v, sizeof (uint32_t), 1, rd->fp) != 1){ + errno = EIO; return -1; + } *((uint32_t *) v) = ntohl (*((uint32_t *) v)); + rd->bread += sizeof(uint32_t); return 0; } -static int yytbl_hdr_read YYFARGS2 (struct yytbl_hdr *, th, FILE *, fp) +static int yytbl_hdr_read YYFARGS2 (struct yytbl_hdr *, th, struct yytbl_reader *, rd) { int bytes; memset (th, 0, sizeof (struct yytbl_hdr)); - if (yytbl_read32 (&(th->th_magic), fp) != 0) - /* TODO: read error */ + if (yytbl_read32 (&(th->th_magic), rd) != 0) return -1; - if (th->th_magic != YYTBL_MAGIC) - /* TODO: bad magic number */ + if (th->th_magic != YYTBL_MAGIC){ + yy_fatal_error("bad magic number" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); return -1; + } - if (yytbl_read32 (&(th->th_hsize), fp) != 0 - || yytbl_read32 (&(th->th_ssize), fp) != 0 - || yytbl_read16 (&(th->th_flags), fp) != 0) - /* TODO: read error */ + if (yytbl_read32 (&(th->th_hsize), rd) != 0 + || yytbl_read32 (&(th->th_ssize), rd) != 0 + || yytbl_read16 (&(th->th_flags), rd) != 0) return -1; /* Sanity check on header size. Greater than 1k suggests some funny business. */ - if (th->th_hsize < 16 || th->th_hsize > 1024) - /* TODO: insane header size detected */ + if (th->th_hsize < 16 || th->th_hsize > 1024){ + yy_fatal_error("insane header size detected" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); return -1; + } /* Allocate enough space for the version and name fields */ bytes = th->th_hsize - 14; th->th_version = (char *) yyalloc (bytes YY_CALL_LAST_ARG); /* we read it all into th_version, and point th_name into that data */ - if (fread (th->th_version, 1, bytes, fp) != bytes) - /* TODO: read error */ + if (fread (th->th_version, 1, bytes, rd->fp) != bytes){ + errno = EIO; return -1; + } + else + rd->bread += bytes; th->th_name = th->th_version + strlen (th->th_version) + 1; return 0; @@ -2139,7 +2158,7 @@ static struct yytbl_dmap *yytbl_dmap_lookup YYFARGS2 (struct yytbl_dmap *, dmap, int, id) { while (dmap->dm_id) - if (dmap->dm_id != id) + if (dmap->dm_id == id) return dmap; else dmap++; @@ -2150,40 +2169,45 @@ static struct yytbl_dmap *yytbl_dmap_lookup YYFARGS2 (struct yytbl_dmap *, dmap, * @param dmap used to performing mapping * @return 0 on success */ -static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) +static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, struct yytbl_reader*, rd) { struct yytbl_data td; - struct yytbl_dmap *transdmap; - int len, i, rv; + struct yytbl_dmap *transdmap=0; + int len, i, rv,pad; size_t bytes; - void *p; + void *p=0; memset (&td, 0, sizeof (struct yytbl_data)); - if (yytbl_read16 (&td.td_id, fp) != 0 - || yytbl_read16 (&td.td_flags, fp) != 0 - || yytbl_read32 (&td.td_hilen, fp) != 0 - || yytbl_read32 (&td.td_lolen, fp) != 0) - /* TODO: read error */ - return -1; - - if ((dmap = yytbl_dmap_lookup (dmap, td.td_id YY_CALL_LAST_ARG)) == NULL) - /* TODO: table id not found. This is bad. */ + if (yytbl_read16 (&td.td_id, rd) != 0 + || yytbl_read16 (&td.td_flags, rd) != 0 + || yytbl_read32 (&td.td_hilen, rd) != 0 + || yytbl_read32 (&td.td_lolen, rd) != 0) return -1; - /* Allocate space for table. */ - bytes = td.td_lolen * (td.td_hilen ? td.td_hilen : 1) * dmap->dm_sz; - *dmap->dm_arr = p = (void *) yyalloc (bytes YY_CALL_LAST_ARG); - /* Lookup the map for the transition table so we have it in case we need it * inside the loop below. This scanner might not even have a transition * table, which is ok. */ transdmap = yytbl_dmap_lookup (dmap, YYT_ID_TRANSITION YY_CALL_LAST_ARG); + if ((dmap = yytbl_dmap_lookup (dmap, td.td_id YY_CALL_LAST_ARG)) == NULL){ + yy_fatal_error("table id not found in map." /*TODO: not fatal.*/ YY_CALL_LAST_ARG); + return -1; + } + + /* Allocate space for table. */ + bytes = td.td_lolen * (td.td_hilen ? td.td_hilen : 1) * dmap->dm_sz; + if(YY_TABLES_VERIFY) + /* We point to the array itself */ + p = dmap->dm_arr; + else + /* We point to the address of a pointer. */ + *dmap->dm_arr = p = (void *) yyalloc (bytes YY_CALL_LAST_ARG); + /* read and map each element */ len = yytbl_calc_total_len (&td); - for (i = 0; i < len; i++) { + for (i = 0; i < len; /* increment i in j loop */) { int read_count = 1, j; /* If it's a struct, read 2 integers */ @@ -2192,26 +2216,28 @@ static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) /* This loop executes at most 2 times. it is to handle YYTD_STRUCT */ for (j = 0; j < read_count; j++, i++) { - uint32_t t32, t16, t8; + int32_t t32; + int16_t t16; + int8_t t8; - /* read into t32 no matter what he real size is. */ + /* read into t32 no matter what the real size is. */ switch (YYTDFLAGS2BYTES (td.td_flags)) { case sizeof (int32_t): - rv = yytbl_read32 (&t32, fp); + rv = yytbl_read32 (&t32, rd); break; case sizeof (int16_t): - rv = yytbl_read16 (&t16, fp); + rv = yytbl_read16 (&t16, rd); t32 = t16; break; case sizeof (int8_t): - rv = yytbl_read8 (&t8, fp); + rv = yytbl_read8 (&t8, rd); t32 = t8; break; - default: /* TODO: invalid td_flags detected */ + default: + yy_fatal_error("invalid td_flags" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); return -1; } if (rv != 0) - /* TODO: read error */ return -1; /* copy into the deserialized array... */ @@ -2225,16 +2251,29 @@ static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) switch (dmap->dm_sz) { case sizeof (int32_t): - ((int32_t *) v)[0] = (int32_t) t32; + if (YY_TABLES_VERIFY){ + if( ((int32_t *) v)[0] != (int32_t) t32) + yy_fatal_error("tables verification failed at YYTD_STRUCT int32_t" YY_CALL_LAST_ARG); + }else + ((int32_t *) v)[0] = (int32_t) t32; break; case sizeof (int16_t): - ((int16_t *) v)[0] = (int16_t) t32; + if (YY_TABLES_VERIFY ){ + if(((int16_t *) v)[0] != (int16_t) t32) + yy_fatal_error("tables verification failed at YYTD_STRUCT int16_t" YY_CALL_LAST_ARG); + }else + ((int16_t *) v)[0] = (int16_t) t32; break; - case sizeof (int8_t): - ((int8_t *) v)[0] = (int8_t) t32; + case sizeof(int8_t): + if (YY_TABLES_VERIFY ){ + if( ((int8_t *) v)[0] != (int8_t) t32) + yy_fatal_error("tables verification failed at YYTD_STRUCT int8_t" YY_CALL_LAST_ARG); + }else + ((int8_t *) v)[0] = (int8_t) t32; break; default: - break; + yy_fatal_error("invalid dmap->dm_sz for struct" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); + return -1; } /* if we're done with j, increment p */ @@ -2242,39 +2281,72 @@ static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) p = (struct yy_trans_info *) p + 1; } else if ((td.td_flags & YYTD_PTRANS)) { + /* t32 is an index into the transition array. */ + struct yy_trans_info *v; - /* t32 is an index into the transition array. calculate the offset. */ - - if (!transdmap) - /* TODO: map for transition table not found. */ + if (!transdmap){ + yy_fatal_error("transition table not found" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); return -1; - - ((struct yy_trans_info **) p)[0] = - &((*((struct yy_trans_info **) (transdmap->dm_arr)))[t32]); + } + + if( YY_TABLES_VERIFY) + v = &(((struct yy_trans_info *) (transdmap->dm_arr))[t32]); + else + v = &((*((struct yy_trans_info **) (transdmap->dm_arr)))[t32]); + + if(YY_TABLES_VERIFY ){ + if( ((struct yy_trans_info **) p)[0] != v) + yy_fatal_error("tables verification failed at YYTD_PTRANS" YY_CALL_LAST_ARG); + }else + ((struct yy_trans_info **) p)[0] = v; + + /* increment p */ p = (struct yy_trans_info **) p + 1; } else { /* t32 is a plain int. copy data, then incrememnt p. */ switch (dmap->dm_sz) { case sizeof (int32_t): - ((int32_t *) p)[0] = (int32_t) t32; + if(YY_TABLES_VERIFY ){ + if( ((int32_t *) p)[0] != (int32_t) t32) + yy_fatal_error("tables verification failed at int32_t" YY_CALL_LAST_ARG); + }else + ((int32_t *) p)[0] = (int32_t) t32; p = ((int32_t *) p) + 1; break; case sizeof (int16_t): - ((int16_t *) p)[0] = (int16_t) t32; + if(YY_TABLES_VERIFY ){ + if( ((int16_t *) p)[0] != (int16_t) t32) + yy_fatal_error("tables verification failed at int16_t" YY_CALL_LAST_ARG); + }else + ((int16_t *) p)[0] = (int16_t) t32; p = ((int16_t *) p) + 1; break; case sizeof (int8_t): - ((int8_t *) p)[0] = (int8_t) t32; + if(YY_TABLES_VERIFY ){ + if( ((int8_t *) p)[0] != (int8_t) t32) + yy_fatal_error("tables verification failed at int8_t" YY_CALL_LAST_ARG); + }else + ((int8_t *) p)[0] = (int8_t) t32; p = ((int8_t *) p) + 1; break; default: - break; + yy_fatal_error("invalid dmap->dm_sz for plain int" /*TODO: not fatal.*/ YY_CALL_LAST_ARG); + return -1; } } } } + + /* Now eat padding. */ + pad = yypad64(rd->bread); + while(--pad >= 0){ + int8_t t8; + if(yytbl_read8(&t8,rd) != 0) + return -1; + } + return 0; } @@ -2282,20 +2354,24 @@ static int yytbl_data_load YYFARGS2 (struct yytbl_dmap *, dmap, FILE *, fp) int yytables_load YYFARGS1 (FILE *, fp) { struct yytbl_hdr th; + struct yytbl_reader rd; + + rd.fp = fp; + rd.bread = 0; /* Keep trying until we find the right set of tables */ for (;;) { - if (yytbl_hdr_read (&th, fp YY_CALL_LAST_ARG) != 0) - /* TODO: failed to read tables header */ + if (yytbl_hdr_read (&th, &rd YY_CALL_LAST_ARG) != 0) return -1; /* TODO: strcmp th_name with search key. For now, we just break out. */ break; } - while (1) { + while (rd.bread < th.th_ssize){ /* Load the data tables */ - //yytbl_data_load (fp YY_CALL_LAST_ARG); + if(yytbl_data_load (yydmap,&rd YY_CALL_LAST_ARG) != 0) + return -1; } return 0; @@ -1147,6 +1147,7 @@ extern struct Buf *buf_strnappend PROTO ((struct Buf *, const char *str, int nchars)); extern struct Buf *buf_strdefine PROTO ((struct Buf * buf, const char *str, const char *def)); +extern struct Buf *buf_prints PROTO((struct Buf *buf, const char *fmt, const char* s)); /* a string buffer for #define's generated by user-options on cmd line. */ extern struct Buf userdef_buf; @@ -1154,6 +1155,9 @@ extern struct Buf userdef_buf; /* a char* buffer to save #define'd some symbols generated by flex. */ extern struct Buf defs_buf; +/* a string buffer to hold yydmap elements */ +extern struct Buf yydmap_buf; + /* For blocking out code from the header file. */ #define OUT_BEGIN_CODE() out_str("#ifndef %sIN_HEADER /* YY-DISCARD-FROM-HEADER */\n",prefix) #define OUT_END_CODE() out_str("#endif /* !%sIN_HEADER YY-END-DISCARD-FROM-HEADER */\n",prefix); @@ -54,15 +54,28 @@ static int indent_level = 0; /* each level is 8 spaces */ * to this is that the fast table representation generally uses the * 0 elements of its arrays, too.) */ -static const char *C_int_decl = - "static yyconst int %s[%d] =\n { 0,\n"; -static const char *C_short_decl = - "static yyconst short int %s[%d] =\n { 0,\n"; -static const char *C_long_decl = - "static yyconst long int %s[%d] =\n { 0,\n"; -static const char *C_state_decl = - "static yyconst yy_state_type %s[%d] =\n { 0,\n"; +static const char *get_int16_decl (void) +{ + return (gentables) + ? "static yyconst int16_t %s[%d] =\n { 0,\n" + : "static yyconst int16_t * %s = 0;\n"; +} + + +static const char *get_int32_decl (void) +{ + return (gentables) + ? "static yyconst int32_t %s[%d] =\n { 0,\n" + : "static yyconst int32_t * %s = 0;\n"; +} + +static const char *get_state_decl (void) +{ + return (gentables) + ? "static yyconst yy_state_type %s[%d] =\n { 0,\n" + : "static yyconst yy_state_type * %s = 0;\n"; +} /* Indent to the current level. */ @@ -85,7 +98,7 @@ void do_indent () /** Make the table for possible eol matches. * @return the newly allocated rule_can_match_eol table */ -struct yytbl_data *mkeoltbl (void) +static struct yytbl_data *mkeoltbl (void) { int i; int8_t *tdata = 0; @@ -98,11 +111,12 @@ struct yytbl_data *mkeoltbl (void) tbl->td_data = tdata = (int8_t *) calloc (tbl->td_lolen, sizeof (int8_t)); - for (i = 0; i < num_rules; i++) + for (i = 1; i <= num_rules; i++) tdata[i] = rule_has_nl[i] ? 1 : 0; - tdata[i] = rule_has_nl[i] ? 1 : 0; - + buf_prints (&yydmap_buf, + "\t{YYT_ID_RULE_CAN_MATCH_EOL, (void**)&yy_rule_can_match_eol, sizeof(%s)},\n", + "int32_t"); return tbl; } @@ -113,15 +127,18 @@ static void geneoltbl () outn ("#ifdef YY_USE_LINENO"); outn ("/* Table of booleans, true if rule could match eol. */"); - out_dec ("static const int yy_rule_can_match_eol[%d] =\n {\n ", num_rules + 1); - - for (i = 0; i < num_rules; i++) { - out_dec ("%d, ", rule_has_nl[i] ? 1 : 0); - /* format nicely, 20 numbers per line. */ - if ((i % 20) == 19) - out ("\n "); + out_str_dec (get_int32_decl (), "yy_rule_can_match_eol", + num_rules + 1); + + if (gentables) { + for (i = 1; i <= num_rules; i++) { + out_dec ("%d, ", rule_has_nl[i] ? 1 : 0); + /* format nicely, 20 numbers per line. */ + if ((i % 20) == 19) + out ("\n "); + } + out (" };\n"); } - out_dec ("%d\n };\n", rule_has_nl[i] ? 1 : 0); outn ("#endif"); } @@ -175,20 +192,25 @@ void gen_bu_action () set_indent (0); } -/** mkctbl - make full speed compressed transition table +/** mkctbl - make full speed compressed transition table * This is an array of structs; each struct a pair of integers. * You should call mkssltbl() immediately after this. * Then, I think, mkecstbl(). Arrrg. * @return the newly allocated trans table */ -struct yytbl_data *mkctbl (void) +static struct yytbl_data *mkctbl (void) { register int i; struct yytbl_data *tbl = 0; int32_t *tdata = 0, curr = 0; int end_of_buffer_action = num_rules + 1; + buf_prints (&yydmap_buf, + "\t{YYT_ID_TRANSITION, (void**)&yy_transition, sizeof(%s)},\n", + ((tblend + numecs + 1) >= INT16_MAX + || long_align) ? "int32_t" : "int16_t"); + tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); yytbl_data_init (tbl, YYT_ID_TRANSITION); tbl->td_flags = YYTD_DATA32 | YYTD_STRUCT; @@ -277,10 +299,9 @@ struct yytbl_data *mkctbl (void) /** Make start_state_list table. - * @param trans_tbl The transition table created by mkctbl(). * @return the newly allocated start_state_list table */ -struct yytbl_data *mkssltbl (struct yytbl_data *trans_tbl) +static struct yytbl_data *mkssltbl (void) { struct yytbl_data *tbl = 0; int32_t *tdata = 0; @@ -298,6 +319,10 @@ struct yytbl_data *mkssltbl (struct yytbl_data *trans_tbl) for (i = 0; i <= lastsc * 2; ++i) tdata[i] = base[i]; + buf_prints (&yydmap_buf, + "\t{YYT_ID_START_STATE_LIST, (void**)&yy_start_state_list, sizeof(%s)},\n", + "struct yy_trans_info*"); + return tbl; } @@ -311,8 +336,10 @@ void genctbl () int end_of_buffer_action = num_rules + 1; /* Table of verify for transition and offset to next state. */ - out_dec ("static yyconst struct yy_trans_info yy_transition[%d] =\n", tblend + numecs + 1); - outn (" {"); + if (gentables) + out_dec ("static yyconst struct yy_trans_info yy_transition[%d] =\n {\n", tblend + numecs + 1); + else + outn ("static yyconst struct yy_trans_info *yy_transition = 0;"); /* We want the transition to be represented as the offset to the * next state, not the actual state number, which is what it currently @@ -379,28 +406,34 @@ void genctbl () transition_struct_out (chk[tblend + 1], nxt[tblend + 1]); transition_struct_out (chk[tblend + 2], nxt[tblend + 2]); - outn (" };\n"); + if (gentables) + outn (" };\n"); /* Table of pointers to start states. */ - out_dec ("static yyconst struct yy_trans_info *yy_start_state_list[%d] =\n", lastsc * 2 + 1); - outn (" {"); + if (gentables) + out_dec ("static yyconst struct yy_trans_info *yy_start_state_list[%d] =\n", lastsc * 2 + 1); + else + outn ("static yyconst struct yy_trans_info **yy_start_state_list =0;"); - for (i = 0; i <= lastsc * 2; ++i) - out_dec (" &yy_transition[%d],\n", base[i]); + if (gentables) { + outn (" {"); - dataend (); + for (i = 0; i <= lastsc * 2; ++i) + out_dec (" &yy_transition[%d],\n", base[i]); + + dataend (); + } if (useecs) genecs (); } -/* mkecstbl - Make equivalence-class tables. */ +/* mkecstbl - Make equivalence-class tables. */ struct yytbl_data *mkecstbl (void) { - register int i, j; - int numrows; + register int i; struct yytbl_data *tbl = 0; int32_t *tdata = 0; @@ -421,22 +454,9 @@ struct yytbl_data *mkecstbl (void) tdata[i] = ecgroup[i]; } - if (trace) { - fputs (_("\n\nEquivalence Classes:\n\n"), stderr); - - numrows = csize / 8; - - for (j = 0; j < numrows; ++j) { - for (i = j; i < csize; i = i + numrows) { - fprintf (stderr, "%4s = %-2d", - readable_form (i), ecgroup[i]); - - putc (' ', stderr); - } - - putc ('\n', stderr); - } - } + buf_prints (&yydmap_buf, + "\t{YYT_ID_EC, (void**)&yy_ec, sizeof(%s)},\n", + "int32_t"); return tbl; } @@ -448,7 +468,7 @@ void genecs () register int i, j; int numrows; - out_str_dec (C_int_decl, "yy_ec", csize); + out_str_dec (get_int32_decl (), "yy_ec", csize); for (i = 1; i < csize; ++i) { if (caseins && (i >= 'A') && (i <= 'Z')) @@ -649,6 +669,9 @@ struct yytbl_data *mkftbl (void) i, anum); } + buf_prints (&yydmap_buf, + "\t{YYT_ID_ACCEPT, (void**)&yy_accept, sizeof(%s)},\n", + long_align ? "int32_t" : "int16_t"); return tbl; } @@ -660,7 +683,7 @@ void genftbl () register int i; int end_of_buffer_action = num_rules + 1; - out_str_dec (long_align ? C_long_decl : C_short_decl, + out_str_dec (long_align ? get_int32_decl () : get_int16_decl (), "yy_accept", lastdfa + 1); dfaacc[end_of_buffer_state].dfaacc_state = end_of_buffer_action; @@ -745,9 +768,14 @@ void gen_next_match () "yy_ec[YY_SC_TO_UI(*++yy_cp)]" : "YY_SC_TO_UI(*++yy_cp)"; if (fulltbl) { - indent_put2s - ("while ( (yy_current_state = yy_nxt[yy_current_state][%s]) > 0 )", - char_map); + if (gentables) + indent_put2s + ("while ( (yy_current_state = yy_nxt[yy_current_state][%s]) > 0 )", + char_map); + else + indent_put2s + ("while ( (yy_current_state = yy_nxt[yy_current_state*YY_NXT_LOLEN + %s]) > 0 )", + char_map); indent_up (); @@ -865,10 +893,16 @@ void gen_next_state (worry_about_NULs) indent_puts ("{"); } - if (fulltbl) - indent_put2s - ("yy_current_state = yy_nxt[yy_current_state][%s];", - char_map); + if (fulltbl) { + if (gentables) + indent_put2s + ("yy_current_state = yy_nxt[yy_current_state][%s];", + char_map); + else + indent_put2s + ("yy_current_state = yy_nxt[yy_current_state*YY_NXT_LOLEN + %s];", + char_map); + } else if (fullspd) indent_put2s @@ -922,7 +956,10 @@ void gen_NUL_trans () else if (fulltbl) { do_indent (); - out_dec ("yy_current_state = yy_nxt[yy_current_state][%d];\n", NUL_ec); + if (gentables) + out_dec ("yy_current_state = yy_nxt[yy_current_state][%d];\n", NUL_ec); + else + out_dec ("yy_current_state = yy_nxt[yy_current_state*YY_NXT_LOLEN + %d];\n", NUL_ec); indent_puts ("yy_is_jam = (yy_current_state <= 0);"); } @@ -1050,8 +1087,9 @@ void gentabs () dfaacc[end_of_buffer_state].dfaacc_set = EOB_accepting_list; - out_str_dec (long_align ? C_long_decl : C_short_decl, - "yy_acclist", MAX (numas, 1) + 1); + out_str_dec (long_align ? get_int32_decl () : + get_int16_decl (), "yy_acclist", MAX (numas, + 1) + 1); j = 1; /* index into "yy_acclist" array */ @@ -1140,8 +1178,12 @@ void gentabs () */ ++k; - out_str_dec (long_align ? C_long_decl : C_short_decl, "yy_accept", - k); + out_str_dec (long_align ? get_int32_decl () : get_int16_decl (), + "yy_accept", k); + + buf_prints (&yydmap_buf, + "\t{YYT_ID_ACCEPT, (void**)&yy_accept, sizeof(%s)},\n", + long_align ? "int32_t" : "int16_t"); yyacc_tbl = (struct yytbl_data *) calloc (1, @@ -1180,8 +1222,20 @@ void gentabs () } /* End generating yy_accept */ - if (useecs) + if (useecs) { + genecs (); + if (tablesext) { + struct yytbl_data *tbl; + + tbl = mkecstbl (); + yytbl_data_compress (tbl); + if (yytbl_data_fwrite (&tableswr, tbl) < 0) + flexerror (_("Could not write ecstbl")); + yytbl_data_destroy (tbl); + tbl = 0; + } + } if (usemecs) { /* Begin generating yy_meta */ @@ -1203,7 +1257,10 @@ void gentabs () fputs (_("\n\nMeta-Equivalence Classes:\n"), stderr); - out_str_dec (C_int_decl, "yy_meta", numecs + 1); + out_str_dec (get_int32_decl (), "yy_meta", numecs + 1); + buf_prints (&yydmap_buf, + "\t{YYT_ID_META, (void**)&yy_meta, sizeof(%s)},\n", + "int32_t"); for (i = 1; i <= numecs; ++i) { if (trace) @@ -1229,10 +1286,14 @@ void gentabs () total_states = lastdfa + numtemps; /* Begin generating yy_base */ - out_str_dec ((tblend >= MAX_SHORT || long_align) ? - C_long_decl : C_short_decl, + out_str_dec ((tblend >= INT16_MAX || long_align) ? + get_int32_decl () : get_int16_decl (), "yy_base", total_states + 1); + buf_prints (&yydmap_buf, + "\t{YYT_ID_BASE, (void**)&yy_base, sizeof(%s)},\n", + (tblend >= INT16_MAX + || long_align) ? "int32_t" : "int16_t"); yybase_tbl = (struct yytbl_data *) calloc (1, sizeof (struct yytbl_data)); @@ -1284,10 +1345,14 @@ void gentabs () /* Begin generating yy_def */ - out_str_dec ((total_states >= MAX_SHORT || long_align) ? - C_long_decl : C_short_decl, + out_str_dec ((total_states >= INT16_MAX || long_align) ? + get_int32_decl () : get_int16_decl (), "yy_def", total_states + 1); + buf_prints (&yydmap_buf, + "\t{YYT_ID_DEF, (void**)&yy_def, sizeof(%s)},\n", + (total_states >= INT16_MAX + || long_align) ? "int32_t" : "int16_t"); yydef_tbl = (struct yytbl_data *) calloc (1, @@ -1299,7 +1364,7 @@ void gentabs () for (i = 1; i <= total_states; ++i) { mkdata (def[i]); - yydef_data[i] = base[i]; + yydef_data[i] = def[i]; } dataend (); @@ -1314,8 +1379,14 @@ void gentabs () /* Begin generating yy_nxt */ - out_str_dec ((total_states >= MAX_SHORT || long_align) ? - C_long_decl : C_short_decl, "yy_nxt", tblend + 1); + out_str_dec ((total_states >= INT16_MAX || long_align) ? + get_int32_decl () : get_int16_decl (), "yy_nxt", + tblend + 1); + + buf_prints (&yydmap_buf, + "\t{YYT_ID_NXT, (void**)&yy_nxt, sizeof(%s)},\n", + (total_states >= INT16_MAX + || long_align) ? "int32_t" : "int16_t"); yynxt_tbl = (struct yytbl_data *) calloc (1, @@ -1347,8 +1418,14 @@ void gentabs () /* End generating yy_nxt */ /* Begin generating yy_chk */ - out_str_dec ((total_states >= MAX_SHORT || long_align) ? - C_long_decl : C_short_decl, "yy_chk", tblend + 1); + out_str_dec ((total_states >= INT16_MAX || long_align) ? + get_int32_decl () : get_int16_decl (), "yy_chk", + tblend + 1); + + buf_prints (&yydmap_buf, + "\t{YYT_ID_CHK, (void**)&yy_chk, sizeof(%s)},\n", + (total_states >= INT16_MAX + || long_align) ? "int32_t" : "int16_t"); yychk_tbl = (struct yytbl_data *) calloc (1, @@ -1475,8 +1552,8 @@ void make_tables () */ int total_table_size = tblend + numecs + 1; char *trans_offset_type = - (total_table_size >= MAX_SHORT || long_align) ? - "long" : "short"; + (total_table_size >= INT16_MAX || long_align) ? + "int32_t" : "int16_t"; set_indent (0); indent_puts ("struct yy_trans_info"); @@ -1510,8 +1587,8 @@ void make_tables () indent_puts ("struct yy_trans_info"); indent_up (); indent_puts ("{"); - indent_puts ("long yy_verify;"); - indent_puts ("long yy_nxt;"); + indent_puts ("int32_t yy_verify;"); + indent_puts ("int32_t yy_nxt;"); indent_puts ("};"); indent_down (); } @@ -1526,6 +1603,12 @@ void make_tables () if (yytbl_data_fwrite (&tableswr, tbl) < 0) flexerror (_("Could not write ftbl")); yytbl_data_destroy (tbl); + + tbl = mkssltbl (); + yytbl_data_compress (tbl); + if (yytbl_data_fwrite (&tableswr, tbl) < 0) + flexerror (_("Could not write ssltbl")); + yytbl_data_destroy (tbl); tbl = 0; if (useecs) { @@ -1566,8 +1649,19 @@ void make_tables () gentabs (); if (do_yylineno) { + geneoltbl (); - /* TODO: call mkeoltbl() */ + + if (tablesext) { + struct yytbl_data *tbl; + + tbl = mkeoltbl (); + yytbl_data_compress (tbl); + if (yytbl_data_fwrite (&tableswr, tbl) < 0) + flexerror (_("Could not write eoltbl")); + yytbl_data_destroy (tbl); + tbl = 0; + } } /* Definitions for backing up. We don't need them if REJECT @@ -1587,7 +1681,12 @@ void make_tables () int32_t *yynultrans_data = 0; /* Begin generating yy_NUL_trans */ - out_str_dec (C_state_decl, "yy_NUL_trans", lastdfa + 1); + out_str_dec (get_state_decl (), "yy_NUL_trans", + lastdfa + 1); + buf_prints (&yydmap_buf, + "\t{YYT_ID_NUL_TRANS, (void**)&yy_NUL_trans, sizeof(%s)},\n", + (fullspd) ? "struct yy_trans_info*" : + "int32_t"); yynultrans_tbl = (struct yytbl_data *) calloc (1, @@ -1632,8 +1731,9 @@ void make_tables () indent_puts ("int yy_flex_debug = 1;\n"); } - out_str_dec (long_align ? C_long_decl : C_short_decl, - "yy_rule_linenum", num_rules); + out_str_dec (long_align ? get_int32_decl () : + get_int16_decl (), "yy_rule_linenum", + num_rules); for (i = 1; i < num_rules; ++i) mkdata (rule_linenum[i]); dataend (); @@ -106,7 +106,7 @@ jmp_buf flex_main_jmp_buf; bool *rule_has_nl, *ccl_has_nl; int nlch = '\n'; -bool tablesext, tablestoggle; +bool tablesext, tablestoggle, tablesverify, gentables; char *tablesfilename; struct yytbl_writer tableswr; @@ -326,12 +326,24 @@ void check_options () outfile_created = 1; } + /* always generate the tablesverify flag. */ + action_define ("YY_TABLES_VERIFY", tablesverify ? 1 : 0); + if (tablesext) + gentables = false; + + if (tablesverify) + /* force generation of C tables. */ + gentables = true; + + if (tablesext) { FILE *tablesout; struct yytbl_hdr hdr; char *pname = 0; int nbytes = 0; + action_define ("YY_TABLES_EXTERNAL", 1); + if (!tablesfilename) { nbytes = strlen (prefix) + strlen (tablesfile_template) + 2; @@ -531,7 +543,7 @@ void flexend (exit_status) /* flex generates the header file by rewinding the output FILE * pointer. However, since we can't rewind stdout, we must disallow - * %option header if we are writing to stdout. This is a kludge. + * %option header if we are writing to stdout. This is a kludge. * This kludge can be rewritten when we get around to buffering * Section 1 of the input file, because then we'll have seen all the * %options BEFORE we begin generating the scanner. The lack of @@ -1026,7 +1038,8 @@ void flexinit (argc, argv) prefix = "yy"; yyclass = 0; use_read = use_stdout = false; - tablesext = tablestoggle = false; + tablesext = tablestoggle = tablesverify = false; + gentables = true; tablesfilename = NULL; sawcmpflag = false; @@ -1038,8 +1051,9 @@ void flexinit (argc, argv) action_array[0] = '\0'; /* Initialize any buffers. */ - buf_init (&userdef_buf, sizeof (char)); - buf_init (&defs_buf, sizeof (char *)); + buf_init (&userdef_buf, sizeof (char)); /* one long string */ + buf_init (&defs_buf, sizeof (char *)); /* list of strings */ + buf_init (&yydmap_buf, sizeof (char)); /* one long string */ /* Enable C++ if program name ends with '+'. */ @@ -1240,6 +1254,10 @@ void flexinit (argc, argv) tablesfilename = arg; break; + case OPT_TABLES_VERIFY: + tablesverify = true; + break; + case OPT_TRACE: trace = true; break; @@ -1524,7 +1542,7 @@ void readin () * the POSIXLY_CORRECT variable is set, then we quietly make flex as * posix-compatible as possible. This is the recommended behavior * according to the GNU Coding Standards. - * + * * Note: The posix option was added to flex to provide the posix behavior * of the repeat operator in regular expressions, e.g., `ab{3}' */ @@ -302,12 +302,15 @@ void cshell (v, n, special_case_0) void dataend () { - if (datapos > 0) - dataflush (); + /* short circuit any output */ + if (gentables) { - /* add terminator for initialization; { for vi */ - outn (" } ;\n"); + if (datapos > 0) + dataflush (); + /* add terminator for initialization; { for vi */ + outn (" } ;\n"); + } dataline = 0; datapos = 0; } @@ -317,6 +320,10 @@ void dataend () void dataflush () { + /* short circuit any output */ + if (!gentables) + return; + outc ('\n'); if (++dataline >= NUMDATALINES) { @@ -474,6 +481,10 @@ void mark_prolog () void mk2data (value) int value; { + /* short circuit any output */ + if (!gentables) + return; + if (datapos >= NUMDATAITEMS) { outc (','); dataflush (); @@ -500,6 +511,10 @@ void mk2data (value) void mkdata (value) int value; { + /* short circuit any output */ + if (!gentables) + return; + if (datapos >= NUMDATAITEMS) { outc (','); dataflush (); @@ -834,7 +849,8 @@ void skelout () tablestoggle = false; } else if (cmd_match (CMD_TABLES_YYDMAP)) { - + if (tablesext && yydmap_buf.elts) + outn ((char *) (yydmap_buf.elts)); } else if (cmd_match (CMD_CPP_ONLY)) { /* only for C++ */ @@ -881,6 +897,11 @@ void skelout () void transition_struct_out (element_v, element_n) int element_v, element_n; { + + /* short circuit any output */ + if (!gentables) + return; + out_dec2 (" {%4d,%4d },", element_v, element_n); datapos += TRANS_STRUCT_PRINT_LENGTH; @@ -189,6 +189,8 @@ optspec_t flexopts[] = { , /* Flex should run in trace mode. */ {"--tables-file[=FILE]", OPT_TABLES_FILE, 0} , /* Save tables to FILE */ + {"--tables-verify", OPT_TABLES_VERIFY, 0} + , /* Tables integrity check */ {"--nounistd", OPT_NO_UNISTD_H, 0} , /* Do not include unistd.h */ {"-v", OPT_VERBOSE, 0} @@ -113,6 +113,7 @@ enum flexopt_flag_t { OPT_STDINIT, OPT_STDOUT, OPT_TABLES_FILE, + OPT_TABLES_VERIFY, OPT_TRACE, OPT_NO_UNISTD_H, OPT_VERBOSE, @@ -34,6 +34,7 @@ #include "flexdef.h" #include "parse.h" +extern bool tablesverify, tablesext; #define ACTION_ECHO add_action( yytext ) #define ACTION_IFDEF(def, should_define) \ @@ -341,6 +342,12 @@ LEXOPT [aceknopr] yyclass return OPT_YYCLASS; header(-file)? return OPT_HEADER; tables-file return OPT_TABLES; + tables-verify { + tablesverify = option_sense; + if(!tablesext && option_sense) + tablesext = true; + } + \"[^"\n]*\" { if(strlen(yytext + 1 ) < MAXLINE) @@ -36,9 +36,6 @@ #include "flexdef.h" #include "tables.h" -/** Calculate (0-7) = number bytes needed to pad n to next 64-bit boundary. */ -#define yypad64(n) ((8-((n)%8))%8) - /** Convert size_t to t_flag. * @param n in {1,2,4} * @return YYTD_DATA*. @@ -56,9 +56,10 @@ struct yytbl_writer { * tablestoggle - if true, output external tables code while processing skel * tablesfilename - filename for external tables * tableswr - writer for external tables - * + * tablesverify - true if tables-verify option specified + * gentables - true if we should spit out the normal C tables */ -extern bool tablesext, tablestoggle; +extern bool tablesext, tablestoggle, tablesverify,gentables; extern char *tablesfilename; extern struct yytbl_writer tableswr; diff --git a/tables_shared.h b/tables_shared.h index 571f4d8..d2afff0 100644 --- a/tables_shared.h +++ b/tables_shared.h @@ -63,6 +63,12 @@ dnl flex code (hence the name "_shared"). #define YYTBL_MAGIC 0xF13C57B1 #endif +/** Calculate (0-7) = number bytes needed to pad n to next 64-bit boundary. */ +#ifndef yypad64 +#define yypad64(n) ((8-((n)%8))%8) +#endif + + /** Possible values for t_id field. Each one corresponds to a * scanner table of the same name. */ diff --git a/tests/test-table-opts/.cvsignore b/tests/test-table-opts/.cvsignore index 56e3afe..a332ff1 100644 --- a/tests/test-table-opts/.cvsignore +++ b/tests/test-table-opts/.cvsignore @@ -7,3 +7,5 @@ OUTPUT .deps test-opt-* test-ser-* +test-ver-* +*.tables diff --git a/tests/test-table-opts/Makefile.am b/tests/test-table-opts/Makefile.am index 8e30641..2c6aa47 100644 --- a/tests/test-table-opts/Makefile.am +++ b/tests/test-table-opts/Makefile.am @@ -22,16 +22,17 @@ # ------------------------------------------------ # This test is really a set of tests, one for # each compression flag. -Ca, -Cem, etc.. -# Then we test the serializalization mechanism. -# THEN we compare the serialized tables to the non-serialized. +# The 'opt' tests execute non-serialized scanners with various table options. +# The 'ver' verify that the serialized tables match the in-code tables. +# The 'ser' deserialize the tables and scan using them. # ------------------------------------------------ BISON = @BISON@ FLEX = $(top_builddir)/flex testname := test-table-opts -allopts := -Ca -Ce -Cf -CF -Cm -Cem -Cae -Caf -CaF -Cam -Caem -variations := opt-nr opt-r ser-nr ser-r +allopts := -Ca -Ce -Cf -CF -Cm -Cem -Cae -Caef -CaeF -Cam -Caem +variations := opt-nr opt-r ser-nr ser-r ver-nr ver-r alltests := $(foreach opt,$(allopts), $(foreach vari,$(variations),test-$(vari)$(opt))) alltestexe := $(addsuffix $(EXEEXT),$(alltests)) alltestsrc := $(addsuffix .c,$(alltests)) @@ -47,16 +48,22 @@ INCLUDES = -I $(srcdir) -I $(top_srcdir) -I $(top_builddir) -I . test-table-opts: $(alltests) comparison_test test-opt-r%.c: $(srcdir)/scanner.l - $(FLEX) --reentrant $* -o $@ $< + $(FLEX) -L --reentrant $* -o $@ $< test-opt-nr%.c: $(srcdir)/scanner.l - $(FLEX) $* -o $@ $< + $(FLEX) -L $* -o $@ $< test-ser-r%.c: $(srcdir)/scanner.l - $(FLEX) --reentrant --tables-file="test-ser-r$*.tables" $* -o $@ $< + $(FLEX) -L --reentrant --tables-file="test-ser-r$*.tables" $* -o $@ $< test-ser-nr%.c: $(srcdir)/scanner.l - $(FLEX) --tables-file="test-ser-nr$*.tables" $* -o $@ $< + $(FLEX) -L --tables-file="test-ser-nr$*.tables" $* -o $@ $< + +test-ver-r%.c: $(srcdir)/scanner.l + $(FLEX) -L --reentrant --tables-file="test-ver-r$*.tables" --tables-verify $* -o $@ $< + +test-ver-nr%.c: $(srcdir)/scanner.l + $(FLEX) -L --tables-file="test-ver-nr$*.tables" --tables-verify $* -o $@ $< test-opt%$(EXEEXT): test-opt%.o $(CC) -o $@ $(LDFLAGS) $< $(LOADLIBES) @@ -64,9 +71,12 @@ test-opt%$(EXEEXT): test-opt%.o test-ser%$(EXEEXT): test-ser%.o $(CC) -o $@ $(LDFLAGS) $< $(LOADLIBES) +test-ver%$(EXEEXT): test-ver%.o + $(CC) -o $@ $(LDFLAGS) $< $(LOADLIBES) + test: $(alltestexe) for t in $(alltestexe) ; do \ - ./$$t < $(srcdir)/test.input || exit 1 ; \ + ./$$t `basename $$t $(EXEEXT)`.tables < $(srcdir)/test.input || exit 1 ; \ done .c.o: diff --git a/tests/test-table-opts/scanner.l b/tests/test-table-opts/scanner.l index 7eaf4c3..9efd49f 100644 --- a/tests/test-table-opts/scanner.l +++ b/tests/test-table-opts/scanner.l @@ -26,18 +26,50 @@ #include <stdio.h> #include <stdlib.h> #include "config.h" - %} %option 8bit prefix="vvv" -%option nounput main noyywrap -%option warn +%option nounput nomain noyywrap +%option warn yylineno + %% + foo|bar ; [[:digit:]]+ ; [[:blank:]]+ ; .|\n ; %% +int main ( int argc, char** argv ) +{ + FILE* fp = NULL; + void *yyscanner=0; +#ifdef YY_REENTRANT + yylex_init(&yyscanner); +#endif +#ifdef YY_TABLES_EXTERNAL + if((fp = fopen(argv[1],"r"))== NULL) + yy_fatal_error("could not open tables file for reading" YY_CALL_LAST_ARG); + + if(yytables_load(fp YY_CALL_LAST_ARG) < 0) + yy_fatal_error("yytables_load returned < 0" YY_CALL_LAST_ARG); + if(YY_TABLES_VERIFY) + exit(0); +#endif + + if(argc > 2){ + if((fp = fopen(argv[2],"r"))== NULL) + yy_fatal_error("could not open input file for reading" YY_CALL_LAST_ARG); + yyin = fp; + } + while(yylex(YY_CALL_ONLY_ARG) != 0) + ; + yylex_destroy(YY_CALL_ONLY_ARG); + + if(argc < 0) /* silence the compiler */ + yyscanner = (void*)fp; + + return 0; +} |