diff options
Diffstat (limited to 'MISC')
55 files changed, 20457 insertions, 0 deletions
diff --git a/MISC/Amiga/README.amiga b/MISC/Amiga/README.amiga new file mode 100644 index 0000000..0efdb93 --- /dev/null +++ b/MISC/Amiga/README.amiga @@ -0,0 +1,72 @@ + +This file, ./MISC/Amiga/README.amiga, describes the necessary steps to get +the FLEX 2.5.1 program up and running on the Commodore AMIGA with the +help of SAS/C++ 6.51 and SMake 6.50. Also it describes the contents of the +subdirectory ./MISC/Amiga, where `.' denotes the root directory of the +standard installation of FLEX 2.5.1. + + +FILES ADDED FOR INSTALLING FLEX 2.5.1 ON THE AMIGA + +The standard distribution of FLEX 2.5.1 is assumed to be installed on +your AMIGA computer. `Installed' means that all source files from the +original archive are present in a root directory (denoted by `.' in what +follows) and, if appropriate, one or more subdirectories, on your machine. + +The original source files are totally left untouched, the necessary changes +are applied in the form of `change files'. For installing FLEX 2.5.1 on +the AMIGA, several additional files come with this patch. + + -----rw-d 2 738 Apr 3 11:49 config.h + -----rw-d 1 169 Apr 3 11:33 libmain.ch + -----rw-d 1 159 Apr 3 11:33 libyywrap.ch + -----rw-d 1 167 Apr 3 11:33 parse.ych + -----rw-d 6 2840 Apr 3 11:34 README.amiga + -----rw-d 11 5503 Apr 3 11:45 smakefile + Dirs:0 Files:6 Blocks:22 Bytes:9576 + + +HOW TO INSTALL FLEX 2.5.1 ON THE AMIGA + +Copy all files from ./MISC/Amiga to the root directory of the FLEX 2.5.1 +distribution and edit `SMakefile' as it instructs you. There shouldn't be +too many changes necessary. Then say "make bootflex". This creates a +preliminary version of FLEX 2.5.1 without using itself. + +WARNING: Don't say "make flex" yet. Any pre-2.5 version of FLEX will fail +on the file `scan.l' due to some new features only present in FLEX 2.5. + +Then say "make flex". At least once the FLEX program created in the first +step will be used. To make sure that everything is alright, finally say +"make check". If you change the code, you should also say "make bigcheck" +for some more thorough testing. + +When you are satisfied with the results, say "make install". This will +copy the `flex' binary, the `libfl.lib' file, and the `FlexLexer.h' header +to the paths specified in the `SMakefile'. + +Finally, you should say "make clean" to remove all intermediate files from +the root directory. "make veryclean" also removes `flex' and `scan.c'. + + +TROUBLE SHOOTING + +FLEX 2.5.1 was ported to the AMIGA and tested with the following setup: + + AMIGA 2000 + GVP G-Force 030/50/50/8 + SAS/C++ 6.51 + SED 2.05 + BISON 1.22 + WMERGE from the CWEB distribution + +Should you encounter problems with this AMIGA patch for FLEX 2.5.1 or +should you have ideas for further improvements, like using GnuMake instead +of SMake, contact the author of this contribution + +Andreas Scherer +Roland-Stra{\ss}e 16 +52070 Aachen +Germany + +<scherer@genesis.informatik.rwth-aachen.de> (Internet) diff --git a/MISC/Amiga/SMakefile b/MISC/Amiga/SMakefile new file mode 100644 index 0000000..28e0dd4 --- /dev/null +++ b/MISC/Amiga/SMakefile @@ -0,0 +1,195 @@ +# @(#) $Header: Makefile.in,v 1.2 94/01/04 14:33:19 vern Exp $ (LBL) + +# If your version of "make" does not define $(MAKE), comment in the +# definition of "MAKE" below. (You only need to do this if you intend +# to do "make bigcheck" or "make dist".) +MAKE = smake + +# Possible values for DEFS: +# +# For flex to always generate 8-bit scanners, add "-DDEFAULT_CSIZE=256" +# to DEFS. +# +# For Vax/VMS, add "-DVMS" to DEFS. +# +# For MS-DOS, add "-DMS_DOS" to DEFS. See the directory MISC/MSDOS for +# additional info. + +CFLAGS = data=far ignore=85 noicons stackextend optimize +DEFS = define=YYBISON=1 define=YY_NEVER_INTERACTIVE=1 +LDFLAGS = noicons +LIBS = + +# Installation targeting. Files will be installed under the tree +# rooted at prefix. flex will be installed in bindir, libfl.lib in +# libdir, FlexLexer.h will be installed in includedir, and the manual +# pages will be installed in mandir with extension manext. +# +# Raw, unformatted troff source will be installed if INSTALLMAN=man, +# nroff preformatted versions will be installed if INSTALLMAN=cat. + +prefix = Programmer:other # Change this for your AMIGA system. +exec_prefix = $(prefix) +bindir = $(exec_prefix)/bin +libdir = $(exec_prefix)/lib +includedir = $(prefix)/include +manext = 1 +mandir = $(prefix)/man/man$(manext) + +# You can define this to be "lex" if you want to replace lex at your site. +FLEX = flex + +INSTALLMAN = man + +SHELL = +srcdir = . +VPATH = + +LN_S = +YACC = bison -y +SED = sed +CC = sc +WMERGE = wmerge # from the CWEB distribution +AR = oml +RANLIB = +INSTALL = copy clone +INSTALL_DATA = $(INSTALL) +INSTALL_PROGRAM = $(INSTALL) + +# You normally do not need to modify anything below this point. +# ------------------------------------------------------------ + +CPPFLAGS = idir=. idir=$(srcdir) $(DEFS) + +.c.o: + $(CC) $(CPPFLAGS) $(CFLAGS) $< + +HEADERS = flexdef.h version.h + +SOURCES = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.y \ + scan.l skel.c sym.c tblcmp.c yylex.c +OBJECTS = ccl.o dfa.o ecs.o gen.o main.o misc.o nfa.o parse.o \ + skel.o sym.o tblcmp.o yylex.o \ + $(libdir)/alloca.o $(libdir)/xmalloc.o + +LIBSRCS = libmain.c libyywrap.c +LIBOBJS = ansilibmain.o ansilibyywrap.o + +LINTSRCS = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.c \ + scan.c skel.c sym.c tblcmp.c yylex.c + +DISTFILES = README NEWS COPYING INSTALL FlexLexer.h \ + configure.in conf.in Makefile.in mkskel.sh skel.c flex.skl \ + $(HEADERS) $(SOURCES) $(LIBSRCS) MISC \ + flex.1 scan.c install.sh mkinstalldirs configure + +DIST_NAME = flex + +# which "flex" to use to generate scan.c from scan.l +FLEX_EXEC = ./$(FLEX) +FLEX_FLAGS = -t $(PERF_REPORT) +COMPRESSION = +PERF_REPORT = -p + +FLEXLIB = libfl.lib + +all: $(FLEX) + +$(FLEX): $(OBJECTS) $(FLEXLIB) scan.o + $(CC) $(CFLAGS) link to $(FLEX) $(LDFLAGS) $(OBJECTS) scan.o $(FLEXLIB) $(LIBS) + +bootflex: $(OBJECTS) $(FLEXLIB) initscan.o + $(CC) $(CFLAGS) link to $(FLEX) $(LDFLAGS) $(OBJECTS) initscan.o $(FLEXLIB) $(LIBS) + +parse.c: ansiparse.y + $(YACC) -d ansiparse.y + $(SED) "/extern char.*malloc/d" <y.tab.c >parse.tmp + copy parse.tmp parse.c + copy y.tab.h parse.h + @delete y.tab.c y.tab.h parse.tmp +ansiparse.y: $(srcdir)/parse.y parse.ych + $(WMERGE) $(srcdir)/parse.y parse.ych ansiparse.y + +parse.h: parse.c + +scan.c: scan.l + $(FLEX_EXEC) $(FLEX_FLAGS) $(COMPRESSION) $(srcdir)/scan.l >scan.tmp + $(SED) s,\"$(srcdir)/scan.l\",\"scan.l\", <scan.tmp >scan.c + @delete scan.tmp + +scan.o: scan.c parse.h flexdef.h config.h +initscan.o: initscan.c parse.h flexdef.h config.h +yylex.o: yylex.c parse.h flexdef.h config.h + +skel.c: flex.skl mkskel.sh + $(SHELL) $(srcdir)/mkskel.sh $(srcdir)/flex.skl >skel.c + +main.o: main.c flexdef.h config.h version.h +ccl.o: ccl.c flexdef.h config.h +dfa.o: dfa.c flexdef.h config.h +ecs.o: ecs.c flexdef.h config.h +gen.o: gen.c flexdef.h config.h +misc.o: misc.c flexdef.h config.h +nfa.o: nfa.c flexdef.h config.h +parse.o: parse.c flexdef.h config.h +skel.o: skel.c flexdef.h config.h +sym.o: sym.c flexdef.h config.h +tblcmp.o: tblcmp.c flexdef.h config.h + +alloca.o: alloca.c + $(CC) $(CPPFLAGS) $(CFLAGS) define=xmalloc=yy_flex_xmalloc alloca.c + +alloca.c: $(srcdir)/MISC/alloca.c + @delete alloca.c + copy $(srcdir)/MISC/alloca.c . + +test: check +check: flex + $(FLEX_EXEC) $(FLEX_FLAGS) $(COMPRESSION) $(srcdir)/scan.l > temp_a + $(SED) s,"$(srcdir)/scan.l","scan.l", < temp_a > temp_b + -diff scan.c temp_b -l10000 -w + @delete temp_? + @echo "Check successful, using COMPRESSION='$(COMPRESSION)'" + +bigcheck: + delete scan.c + $(MAKE) COMPRESSION=-C check + delete scan.c + $(MAKE) COMPRESSION=-Ce check + delete scan.c + $(MAKE) COMPRESSION=-Cm check + delete scan.c + $(MAKE) COMPRESSION=-Cfea check + delete scan.c + $(MAKE) COMPRESSION=-CFer check + delete scan.c + $(MAKE) COMPRESSION=-l PERF_REPORT= check + delete scan.c + $(MAKE) + @echo "All checks successful" + +$(FLEXLIB): $(LIBOBJS) + $(AR) $(FLEXLIB) R $(LIBOBJS) + +$(FLEX).man: flex.1 # SMAKE can't `cd', sorry. And, I don't have nroff. +# cd $(srcdir), nroff -man flex.1 >$(FLEX).man + +install: $(FLEX) $(FLEXLIB) + $(INSTALL_PROGRAM) $(FLEX) $(bindir)/$(FLEX) +# @delete $(bindir)/$(FLEX)++ + $(INSTALL_DATA) $(FLEXLIB) $(libdir)/libfl.lib + $(INSTALL_DATA) $(srcdir)/FlexLexer.h $(includedir)/FlexLexer.h + +ansilibmain.o: ansilibmain.c +ansilibmain.c: libmain.c libmain.ch + $(WMERGE) libmain.c libmain.ch ansilibmain.c +ansilibyywrap.o: ansilibyywrap.c +ansilibyywrap.c: libyywrap.c libyywrap.ch + $(WMERGE) libyywrap.c libyywrap.ch ansilibyywrap.c + +clean: + -delete parse.(c|h) ansi\#? \#?.(bak|o|lnk) \ + alloca.c lex.yy.(c|cc) $(FLEXLIB) + +veryclean: clean + -delete $(FLEX) scan.c diff --git a/MISC/Amiga/config.h b/MISC/Amiga/config.h new file mode 100644 index 0000000..1c02a3c --- /dev/null +++ b/MISC/Amiga/config.h @@ -0,0 +1,25 @@ +/* $Header: /home/daffy/u0/vern/flex/RCS/conf.in,v 1.2 95/01/09 12:11:51 vern Exp $ */ + +/* Define to empty if the keyword does not work. */ +#undef const + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +#undef size_t + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define if you have the <malloc.h> header file. */ +#undef HAVE_MALLOC_H + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define if you have <alloca.h> and it should be used (not on Ultrix). */ +#undef HAVE_ALLOCA_H + +/* Define if platform-specific command line handling is necessary. */ +#undef NEED_ARGV_FIXUP diff --git a/MISC/Amiga/libmain.ch b/MISC/Amiga/libmain.ch new file mode 100644 index 0000000..e0f5b40 --- /dev/null +++ b/MISC/Amiga/libmain.ch @@ -0,0 +1,8 @@ +Changes for LIBMAIN.C 2.4.7 by Andreas Scherer, January 19, 1995. +Modified for LIBMAIN.C 2.5.1, April 3, 1995. + +@x l.5 +extern int yylex(); +@y +extern int yylex(void); +@z diff --git a/MISC/Amiga/libyywrap.ch b/MISC/Amiga/libyywrap.ch new file mode 100644 index 0000000..b2d6a3f --- /dev/null +++ b/MISC/Amiga/libyywrap.ch @@ -0,0 +1,8 @@ +Changes for LIBYYWRAP.C 2.4.7 by Andreas Scherer, January 19, 1995. +Modified for LIBYYWRAP.C 2.5.1, April 3, 1995. + +@x l.5 +int yywrap() +@y +int yywrap(void) +@z diff --git a/MISC/Amiga/parse.ych b/MISC/Amiga/parse.ych new file mode 100644 index 0000000..5671a5b --- /dev/null +++ b/MISC/Amiga/parse.ych @@ -0,0 +1,8 @@ +Changes for PARSE.Y 2.4.7 by Andreas Scherer, January 20, 1995. +Modified for PARSE.Y 2.5.1, April 3, 1995. + +@x l.60 +char *alloca (); +@y +char *alloca(unsigned int); +@z diff --git a/MISC/Atari/Atari.patches b/MISC/Atari/Atari.patches new file mode 100644 index 0000000..dc04263 --- /dev/null +++ b/MISC/Atari/Atari.patches @@ -0,0 +1,911 @@ +(Message inbox:32) +Date: Mon, 03 Jul 89 21:15:32 CET +From: V61%DHDURZ1.BITNET@lbl.gov +Subject: Flex, bug fix, improvments, patches for Minix & TOS +To: vern@lbl-csam.arpa + +At first I have to thank you for your wonderful program. I had ported the +old version to OS9,TOS (Atari ST) and Minix and the new version 2.1 Beta +to Minix and TOS. + +While porting and using flex I detected a bug and made some improvements. +I have included a shared, compressed and uuencoded file contaning all cdiffs +and additional files (Sorry, but I'm on EBCDIC-Bitnet) and a short discussion +of the changes. Even some of the TOS specific changes might be of general +interest ! + +I posted these cdiffs to the minix discussion group, but I think it's up +to you to post them to the unix-sources group. If you plan to post even +the TOS compiler specific patches please contact me because there might be +further compiler (P.D.) additions. If you have an interest I could also +port the new version to OS9 -- this is a little bit more difficult, because +OS9 uses CR as end of line character (the EOL char. is coded into the +initscan.c tables,...). It is necessary to change all occurences of '\n' to +macros and variables and it's useful to add a new -n options (see commented +line in main.c) + + + +The changes: (1.7.89 RAL) + + - Bug fix: The original flex didn't like trailing spaces in exclusive start + condition lists ! If you add an trailing space to line 68 in scan.l + + "%x SECT2 SECT2PROLOG SECT3 CODEBLOCK PICKUPDEF SC CARETISBOL NUM QUOTE " + + you get a misleading error message: + + "Syntax error at line 69: bad start condition list" + + This bug can either be fixed in parse.y or in scan.l . I have chosen the + last because there the fix is minimal: Just change the rule (line 128) + + "\n" to "{OPTWS}\n" + + - Enhancements: + - new option "-?" that provides some help information about the other + flags (main.c) + - new option "-aTMPPATH" that allows a redefinition of the standard + path for the temporary file (this might be rather large >200k if + F,f options are selected). (main.c, flexdef.h (l.376)) + - hexdump of illegal characters -- this proved to be a useful debugging + tool especialy if invisible control characters occur which weren't + covered by the rules. (scan.l fprintf statement line 129,...) + + - Patches due to TOS + - General: TOS uses CR,LF as end of line char., Flex wants only a single + LF as EOL char. Therefore all I/O must be translated using f* calls. + This is done everywhere besides the YY_INPUT macro (flex.skel (scan.c), + line 31) that uses a low level 'read'. This should be definitly changed + to fread, so that all I/O calls occur on the same level. + - the short action_file_name has been "flexXXXXXX.tmp", but that's too + much for TOS,MSDOS ! I changed it to "flexXXXX.tmp" in main.c (patch + covered by the -a option additions) + - some compilers don't like external names that are ambiguous within + the first 8 characters. I defined macros that change all these long + external names to names that are unique within the first 8 characters. + Just define SHORT_EXTERNAL_NAMES to use this feature (flexdef.h) + - some statement changes that some compiler don't like: + typedef *xxx[] -> typedef **xxx (flexdef.h.,l.308) + "/*" -> '/','*' within a comment in (scan.l, l.209) + - changed short "lexyy.c" to "lex_yy.c" what's more similar to the unix + "lex.yy.c" (main.c). + - a few further really compiler dependent changes provided with + #ifdef ATARI && LATTICE res. TURBO braces. + + - Additional TOS only files + - Makefile.tos: Common makefile for all TOS compilers. If you add further + ones please email me the new makefile. + - flex.lnk: Lattice - GST linker argument extension file + - flex.tlk: Turbo linker argument extension file + + +Additional remarks: + +I didn't add a new initscan.c (= flex -ist scan.l). The current one is good +enough for a first compilation. With this first version of flex you can +rebuild your own scan.c and the final flex version ! + +Minix ST : + - I had to "chmem =70000 cv" (>50000) to get flex linked + - 'memset' (PC 1.3, EFTH40,...) is necessary + - chmem =90000 flex may be sufficient + +Minix PC : + It should be possible to port Flex to Minix PC. The current sizes of flex + are: + Minix ST (ACK) Lattice (TOS) Turbo (TOS) + + size 75300 83305 57957 + compilation time 22' 15' 3'40" + flex -is scan.l 1'49" 43" 30" + + The Minix ST size includes the bad generated code using only a subset of + the 68000 commands, long addresses only and a huge relocation table. + Therefore the PC size will be <64 k ! More serious is the fact that I had + to chmem =90000 flex to get scan.l converted to scan.c . But I never saw + a more complex lex source than scan.l -- so it should be possible to + reduce some array sizes without limitation for all day usage. + + No one volunteered yet for a Minix PC port -- but if someone will try it + I would provide him with a new scan.c and some hints. + +TOS: + Don't forget to adapt the flexskel path within flexdef.h ! + + +Bitnet: V61@DHDURZ1 Ronald Lamprecht +UUCP: ...!unido!DHDURZ1.bitnet!V61 Theoretische Physik +ARPAnet: V61%DHDURZ1.BITNET@CUNYVM.CUNY.EDU (Heidelberg, West Germany) +(Message inbox:36) +Date: Wed, 05 Jul 89 21:16:07 CET +From: V61%DHDURZ1.BITNET@csa2.lbl.gov +Subject: Re: What is TOS +To: vern@lbl-csam.arpa + +TOS is the name of the Atari ST operating system that is very similar +to MSDOS (Both use CR,LF as end of line character). Therefore I have +been astonished that no EOL convertion porblems occur within MSDOS. + +I have been aware of the double buffering when changing read to fread and I +accept your argument of a possible slow down. But if you integrate the other +Atari - TOS changes, please insert a + #ifdef ATARI + fread .... + #else + read .... + #endif +in flex.skel . + +Bitnet: V61@DHDURZ1 Ronald Lamprecht +UUCP: ...!unido!DHDURZ1.bitnet!V61 Theoretische Physik +ARPAnet: V61%DHDURZ1.BITNET@CUNYVM.CUNY.EDU (Heidelberg, West Germany) + + + + +echo x - Makefile_cdiff +sed '/^X/s///' > Makefile_cdiff << '/' +X*** Src_2.1/Makefile Thu Jun 28 00:06:42 1989 +X--- Makefile Thu Jul 3 02:12:48 1989 +X*************** +X*** 5,10 **** +X--- 5,11 ---- +X # Porting considerations: +X # +X # For System V Unix machines, add -DSYS_V to CFLAGS. +X+ # For Minix (ST), add -DSYS_V to CFLAGS +X # For Vax/VMS, add -DSYS_V to CFLAGS. +X # For MS-DOS, add "-DMS_DOS -DSYS_V" to CFLAGS. Create \tmp if not present. +X # You will also want to rename flex.skel to something with a three +X*************** +X*** 21,28 **** +X SKELETON_DIR = /usr/local/lib +X SKELETON_FILE = flex.skel +X SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_DIR)/$(SKELETON_FILE)\" +X! CFLAGS = -O +X! LDFLAGS = -s +X +X FLEX_FLAGS = +X FLEX = ./flex +X--- 22,29 ---- +X SKELETON_DIR = /usr/local/lib +X SKELETON_FILE = flex.skel +X SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_DIR)/$(SKELETON_FILE)\" +X! CFLAGS = -O -DSYS_V +X! LDFLAGS = +X +X FLEX_FLAGS = +X FLEX = ./flex +X*************** +X*** 57,63 **** +X yylex.c +X +X flex : $(FLEXOBJS) +X! $(CC) $(CFLAGS) -o flex $(LDFLAGS) $(FLEXOBJS) +X +X first_flex: +X cp initscan.c scan.c +X--- 58,65 ---- +X yylex.c +X +X flex : $(FLEXOBJS) +X! $(CC) $(CFLAGS) -o flex $(FLEXOBJS) $(LDFLAGS) +X! chmem =150000 flex +X +X first_flex: +X cp initscan.c scan.c +/ +echo x - flex.skel_cdif +sed '/^X/s///' > flex.skel_cdif << '/' +X*** Src_2.1/flex.skel Thu Jun 28 00:19:20 1989 +X--- flex.skel Thu Jul 2 22:18:31 1989 +X*************** +X*** 28,34 **** +X * is returned in "result". +X */ +X #define YY_INPUT(buf,result,max_size) \ +X! if ( (result = read( fileno(yyin), buf, max_size )) < 0 ) \ +X YY_FATAL_ERROR( "read() in flex scanner failed" ); +X #define YY_NULL 0 +X #define yyterminate() return ( YY_NULL ) +X--- 28,34 ---- +X * is returned in "result". +X */ +X #define YY_INPUT(buf,result,max_size) \ +X! if ( (result = fread( buf,1, max_size, yyin )) < 0 ) \ +X YY_FATAL_ERROR( "read() in flex scanner failed" ); +X #define YY_NULL 0 +X #define yyterminate() return ( YY_NULL ) +/ +echo x - flexdef.h_cdif +sed '/^X/s///' > flexdef.h_cdif << '/' +X*** Src_2.1/flexdef.h Thu Jun 28 00:43:27 1989 +X--- flexdef.h Thu Jul 3 02:45:50 1989 +X*************** +X*** 26,31 **** +X--- 26,40 ---- +X +X /* @(#) $Header: flexdef.h,v 2.0 89/06/20 15:49:50 vern Locked $ (LBL) */ +X +X+ #ifdef ATARI +X+ #define SYS_V +X+ #define abs(x) ((x) < 0 ? -(x) : (x)) +X+ #define SHORT_FILE_NAMES +X+ #ifdef TURBO +X+ #define SHORT_EXTERNAL_NAMES +X+ #endif +X+ #endif +X+ +X #ifndef FILE +X #include <stdio.h> +X #endif +X*************** +X*** 41,47 **** +X #endif +X +X #ifndef VMS +X! char *memset(); +X #else +X /* memset is needed for old versions of the VMS C runtime library */ +X #define memset(s, c, n) \ +X--- 50,58 ---- +X #endif +X +X #ifndef VMS +X! #ifndef ATARI && TURBO +X! char *memset(); +X! #endif +X #else +X /* memset is needed for old versions of the VMS C runtime library */ +X #define memset(s, c, n) \ +X*************** +X*** 81,91 **** +X--- 92,129 ---- +X #define true 1 +X #define false 0 +X +X+ #ifdef ATARI +X+ #define DEFAULT_SKELETON_FILE "D:\\include\\flexskel" +X+ #endif +X+ +X +X #ifndef DEFAULT_SKELETON_FILE +X #define DEFAULT_SKELETON_FILE "flex.skel" +X #endif +X +X+ #ifdef SHORT_EXTERNAL_NAMES +X+ /* avoid long external names that are ambiguous within the first 8 characters */ +X+ #define current_mns c__mns +X+ #define current_max_rules c__max_rules +X+ #define current_max_state_type c__max_state_type +X+ #define current_max_scs c__max_scs +X+ #define current_max_dfa_size c__max__size +X+ #define current_max_xpairs c__max_xpairs +X+ #define current_max_template_xpairs c__max_template_xpairs +X+ #define current_max_dfas c__max_dfas +X+ #define current_maxccls c__maxccles +X+ #define current_max_ccl_tbl_size c__max_ccl_tbl_size +X+ #define indent_puts ind_puts +X+ #define indent_put2s ind_put2s +X+ #define gen_next_compressed_state gen_n_comressed_state +X+ #define gen_next_match gen_n_match +X+ #define gen_next_state gen_n_state +X+ #define variable_trailing_context_rules var_tr_context_rules +X+ #define variable_trailing_rule var_tr_rule +X+ #define backtrack_report backtr_report +X+ #define backtrack_file backtr_file +X+ #endif +X+ +X /* special chk[] values marking the slots taking by end-of-buffer and action +X * numbers +X */ +X*************** +X*** 305,311 **** +X int int_val; +X } ; +X +X! typedef struct hash_entry *hash_table[]; +X +X #define NAME_TABLE_HASH_SIZE 101 +X #define START_COND_HASH_SIZE 101 +X--- 343,349 ---- +X int int_val; +X } ; +X +X! typedef struct hash_entry **hash_table; +X +X #define NAME_TABLE_HASH_SIZE 101 +X #define START_COND_HASH_SIZE 101 +X*************** +X*** 372,378 **** +X extern int datapos, dataline, linenum; +X extern FILE *skelfile, *yyin, *temp_action_file, *backtrack_file; +X extern char *infilename; +X! extern char action_file_name[]; +X +X +X /* variables for stack of states having only one out-transition: +X--- 410,416 ---- +X extern int datapos, dataline, linenum; +X extern FILE *skelfile, *yyin, *temp_action_file, *backtrack_file; +X extern char *infilename; +X! extern char *action_file_name; +X +X +X /* variables for stack of states having only one out-transition: +/ +echo x - main.c_cdiff +sed '/^X/s///' > main.c_cdiff << '/' +X*** Src_2.1/main.c Thu Jun 28 00:30:39 1989 +X--- main.c Thu Jul 3 02:27:47 1989 +X*************** +X*** 81,96 **** +X FILE *temp_action_file; +X FILE *backtrack_file; +X int end_of_buffer_state; +X! #ifndef SHORT_FILE_NAMES +X! char action_file_name[] = "/tmp/flexXXXXXX"; +X! #else +X! char action_file_name[] = "flexXXXXXX.tmp"; +X! #endif +X! +X #ifndef SHORT_FILE_NAMES +X static char outfile[] = "lex.yy.c"; +X #else +X! static char outfile[] = "lexyy.c"; +X #endif +X static int outfile_created = 0; +X +X--- 81,91 ---- +X FILE *temp_action_file; +X FILE *backtrack_file; +X int end_of_buffer_state; +X! char *action_file_name; +X #ifndef SHORT_FILE_NAMES +X static char outfile[] = "lex.yy.c"; +X #else +X! static char outfile[] = "lex_yy.c"; +X #endif +X static int outfile_created = 0; +X +X*************** +X*** 328,333 **** +X--- 323,329 ---- +X { +X int i, sawcmpflag, use_stdout; +X char *arg, *skelname = NULL, *flex_gettime(), clower(), *mktemp(); +X+ char *tmp_action =(char *)0, *malloc(); +X +X printstats = syntaxerror = trace = spprdflt = interactive = caseins = false; +X backtrack_report = performance_report = ddebug = fulltbl = fullspd = false; +X*************** +X*** 349,354 **** +X--- 345,355 ---- +X for ( i = 1; arg[i] != '\0'; ++i ) +X switch ( arg[i] ) +X { +X+ case 'a': +X+ if ( i != 1 ) +X+ flexerror( "-a flag must be given separately" ); +X+ tmp_action = &arg[i+1]; +X+ goto get_next_arg; +X case 'b': +X backtrack_report = true; +X break; +X*************** +X*** 445,452 **** +X printstats = true; +X break; +X +X! default: +X! lerrif( "unknown flag %c", (int) arg[i] ); +X break; +X } +X +X--- 446,458 ---- +X printstats = true; +X break; +X +X! case '?': +X! flexinfo(0); +X! break; +X! +X! default: +X! fprintf(stderr,"flex : unknown flag %c\n", (int) arg[i] ); +X! flexinfo(1); +X break; +X } +X +X*************** +X*** 454,459 **** +X--- 460,493 ---- +X ; +X } +X +X+ +X+ /* if you change the default tmp file names don't forget to change the +X+ initialization for i, too ! +X+ +X+ */ +X+ #ifndef SHORT_FILE_NAMES +X+ i = 10; +X+ if (!tmp_action) i += 5; +X+ #else +X+ i = 12; +X+ #endif +X+ if (tmp_action) +X+ i += strlen(tmp_action) + 1; +X+ if((action_file_name = malloc(i+1)) == (char *)0) +X+ flexerror("No memory for action_file_name"); +X+ *action_file_name = (char) NULL; +X+ if (tmp_action) +X+ strcat(action_file_name,tmp_action); +X+ #ifndef SHORT_FILE_NAMES +X+ else +X+ strcat(action_file_name,"/tmp"); +X+ strcat(action_file_name,"/flexXXXXXX"); +X+ #else +X+ if (tmp_action) +X+ strcat(action_file_name,"/"); +X+ strcat(action_file_name,"flexXXXX.tmp"); +X+ #endif +X+ +X if ( (fulltbl || fullspd) && usemecs ) +X flexerror( "full table and -cm don't make sense together" ); +X +X*************** +X*** 520,526 **** +X if ( (skelfile = fopen( skelname, "r" )) == NULL ) +X lerrsf( "can't open skeleton file %s", skelname ); +X +X! (void) mktemp( action_file_name ); +X +X if ( (temp_action_file = fopen( action_file_name, "w" )) == NULL ) +X lerrsf( "can't open temporary action file %s", action_file_name ); +X--- 554,562 ---- +X if ( (skelfile = fopen( skelname, "r" )) == NULL ) +X lerrsf( "can't open skeleton file %s", skelname ); +X +X! #ifndef ATARI && LATTICE +X! (void) mktemp( action_file_name ); +X! #endif +X +X if ( (temp_action_file = fopen( action_file_name, "w" )) == NULL ) +X lerrsf( "can't open temporary action file %s", action_file_name ); +X*************** +X*** 566,571 **** +X--- 602,640 ---- +X } +X +X +X+ flexinfo(status) +X+ int status; +X+ { +X+ fprintf(stderr,"Syntax : FLEX inp_file\n"); +X+ fprintf(stderr,"Function: fast lexical analyzer generator V%s\n",flex_version); +X+ fprintf(stderr,"Options : a dir_path : directory path for temporary files\n"); +X+ fprintf(stderr," - b : generate backtracking information to lex.backtrack\n"); +X+ fprintf(stderr," - c : compressed table, no equiv., no meta equiv.classes\n"); +X+ fprintf(stderr," e : equivalence classes\n"); +X+ fprintf(stderr," F : fast table\n"); +X+ fprintf(stderr," |f : full table\n"); +X+ fprintf(stderr," |m : meta equivalence classes\n"); +X+ fprintf(stderr," - d : generate debugging scanner\n"); +X+ fprintf(stderr," - F : fast table\n"); +X+ fprintf(stderr," - f : full (not compressed) table\n"); +X+ fprintf(stderr," - I : generate interactive scanner\n"); +X+ fprintf(stderr," - i : generate case-insensitive scanner\n"); +X+ fprintf(stderr," - L : supress #line directives\n"); +X+ /* fprintf(stderr," - n hexnum : generate scanner using <hexnum> as newline char.\n");*/ +X+ fprintf(stderr," - p : generate performance report to stderr\n"); +X+ fprintf(stderr," - S skeleton_path : file path for skeleton file\n"); +X+ fprintf(stderr," - s : suppress echo of unmatched scanner input to stdout\n"); +X+ fprintf(stderr," - T : run flex in trace mode\n"); +X+ #ifdef ATARI +X+ fprintf(stderr," - t : place result on stdout instead of lex_yy.c\n"); +X+ #else +X+ fprintf(stderr," - t : place result on stdout instead of lex.yy.c\n"); +X+ #endif +X+ fprintf(stderr," - v : print statistics of generated scanner\n"); +X+ fprintf(stderr," default = -cem\n"); +X+ exit(status); +X+ } +X+ +X /* readin - read in the rules section of the input file(s) +X * +X * synopsis +/ +echo x - scan.l_cdiff +sed '/^X/s///' > scan.l_cdiff << '/' +X*** Src_2.1/scan.l Thu Jun 30 19:42:00 1989 +X--- scan.l Thu Jul 2 22:19:26 1989 +X*************** +X*** 125,132 **** +X +X {SCNAME} RETURNNAME; +X ^{OPTWS}\n ++linenum; /* allows blank lines in section 1 */ +X! \n ++linenum; return ( '\n' ); +X! . synerr( "illegal character" ); BEGIN(RECOVER); +X +X +X <C_COMMENT>"*/" ECHO; BEGIN(0); +X--- 125,136 ---- +X +X {SCNAME} RETURNNAME; +X ^{OPTWS}\n ++linenum; /* allows blank lines in section 1 */ +X! {OPTWS}\n ++linenum; return ( '\n' ); +X! . { +X! synerr( "illegal character" ); +X! fprintf(stderr,"Char : $%x\n",yytext[yyleng-1]); +X! BEGIN(RECOVER); +X! } +X +X +X <C_COMMENT>"*/" ECHO; BEGIN(0); +X*************** +X*** 206,212 **** +X <SECT2>^{OPTWS}\n ++linenum; /* allow blank lines in section 2 */ +X +X /* this horrible mess of a rule matches indented lines which +X! * do not contain "/*". We need to make the distinction because +X * otherwise this rule will be taken instead of the rule which +X * matches the beginning of comments like this one +X */ +X--- 210,216 ---- +X <SECT2>^{OPTWS}\n ++linenum; /* allow blank lines in section 2 */ +X +X /* this horrible mess of a rule matches indented lines which +X! * do not contain '/','*'. We need to make the distinction because +X * otherwise this rule will be taken instead of the rule which +X * matches the beginning of comments like this one +X */ +/ +echo x - Makefile.tos +sed '/^X/s///' > Makefile.tos << '/' +X# make file for "flex" tool +X +X# @(#) $Header: Makefile,v 2.3 89/06/20 17:27:12 vern Exp $ (LBL) +X +X# Porting considerations: +X# +X# For System V Unix machines, add -DSYS_V to CFLAGS. +X# For Vax/VMS, add -DSYS_V to CFLAGS. +X# For MS-DOS, add "-DMS_DOS -DSYS_V" to CFLAGS. Create \tmp if not present. +X# You will also want to rename flex.skel to something with a three +X# character extension, change SKELETON_FILE below appropriately, +X# For Amiga, add "-DAMIGA -DSYS_V" to CFLAGS. +X# +X# A long time ago, flex was successfully built using Microsoft C and +X# the following options: /AL, /stack:10000, -LARGE, -Ml, -Mt128, -DSYS_V +X +X +X# the first time around use "make first_flex" +X +X# The following definitions must be set according to your compiler - +X# examples for a Lattice Compiler with GST assembler and TURBO C with +X# assembler are provided below and must just be updated (don't forget to +X# update the linker argument extension files (*.lnk,*.tlk), too) : +X# +X#CCPATH = path to compiler directory without trailing \ +X#CHPATH = path to header files without trailing \ +X#CC = filename of the compiler +X#CFLAGS = compiler option flags +X#CIEXT = extension of C sources that should be used for input filenames +X#ASMPATH = path to assembler directory without trailing \ +X#ASM = filename of the assembler +X#AFLAGS = assembler option flags +X#AIEXT = extension of assembler sources that should be used for assembler +X# input filenames +X#AEXT = general assembler filename extension +X#LNKPATH = path to linker directory without trailing \ +X#LINK = filename of the linker +X#LFLAG0 = first option (full pathname of C startupcode) +X#LFLAG1 = further options + option flag for argument extension filename +X#LFLAG2 = further options + option flag for output-filename +X#LNKEXT = extension of linker argument extension file +X#OIEXT = extension of objects that should be used for linker input files +X#OEXT = general object file extension +X +X# Lattice definitions +XCCPATH = d:\latt +XCHPATH = d:\latt\include +XCC = lc.ttp +XCFLAGS = -h -n -cw -cc -i$(CHPATH)\ -g$(CCPATH)\ -dLATTICE -dATARI +XCIEXT = +XASMPATH = d:\gst +XASM = assem.ttp +XAFLAGS = -nolist -errors errors.out +XAIEXT = +XAEXT = .asm +XLNKPATH = d:\gst +XLINK = ld.ttp +XLFLAG0 = +XLFLAG1 = -with +XLFLAG2 = -nolist -sec -mem 200 -prog +XLNKEXT = .lnk +XOIEXT = +XOEXT = .bin +X +X# Turbo definitions +X#CCPATH = d:\turbo +X#CHPATH = d:\turbo\include +X#CC = tcc.prg +X#CFLAGS = -GJMPRZ -H=$(CHPATH)\ -w- -DTURBO -DATARI +X#CIEXT = .c +X#ASMPATH = d:\turbo +X#ASM = mas.prg +X#AFLAGS = +X#AIEXT = .s +X#AEXT = .s +X#LNKPATH = d:\turbo +X#LINK = tlink.ttp +X#LFLAG0 = $(LNKPATH)\lib\tcstart.o +X#LFLAG1 = -I= +X#LFLAG2 = -O= +X#LNKEXT = .tlk +X#OIEXT = .o +X#OEXT = .o +X +X# Other definitions +X# (not used for Atari because of short argument string - defined in flexdef.h +X +XSKELETON_DIR = /usr/lib +XSKELETON_FILE = flex.skel +XSKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_DIR)/$(SKELETON_FILE)\" +X +X +XFLEX = d:\etc\flex.ttp +XFLEX_FLAGS = -is +XYACC = d:\etc\yacc.ttp +XRENAME = d:\bin\rename +XDEL = d:\bin\del +X +X# Internal definitions +XLNK = $(LNKPATH)\$(LINK) +X +XFLEXOBJS = \ +X ccl$(OEXT) \ +X dfa$(OEXT) \ +X ecs$(OEXT) \ +X gen$(OEXT) \ +X main$(OEXT) \ +X misc$(OEXT) \ +X nfa$(OEXT) \ +X parse$(OEXT) \ +X scan$(OEXT) \ +X sym$(OEXT) \ +X tblcmp$(OEXT) \ +X yylex$(OEXT) +X +XFLEX_C_SOURCES = \ +X ccl.c \ +X dfa.c \ +X ecs.c \ +X gen.c \ +X main.c \ +X misc.c \ +X nfa.c \ +X parse.c \ +X scan.c \ +X sym.c \ +X tblcmp.c \ +X yylex.c +X +Xflex : $(FLEXOBJS) +X $(LNK) $(LFLAG0) $(LFLAG1)flex$(LNKEXT) $(LFLAG2)flex.ttp +X +Xfirst_flex: +X cp initscan.c scan.c +X make $(MFLAGS) flex +X +Xparse.h parse.c : parse.y +X $(YACC) -d parse.y +X $(DEL) parse.c +X $(RENAME) y_tab.c parse.c +X $(DEL) parse.h +X $(RENAME) y_tab.h parse.h +X +Xscan.c : scan.l +X $(FLEX) $(FLEX_FLAGS) scan.l +X $(RENAME) lex_yy.c scan.c +X +Xscan$(OEXT) : scan.c parse.h flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) scan$(CIEXT) +X +Xmain$(OEXT) : main.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) main$(CIEXT) +X +Xccl$(OEXT) : ccl.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) ccl$(CIEXT) +X +Xdfa$(OEXT) : dfa.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) dfa$(CIEXT) +X +Xecs$(OEXT) : ecs.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) ecs$(CIEXT) +X +Xgen$(OEXT) : gen.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) gen$(CIEXT) +X +Xmisc$(OEXT) : misc.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) misc$(CIEXT) +X +Xnfa$(OEXT) : nfa.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) nfa$(CIEXT) +X +Xparse$(OEXT) : parse.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) parse$(CIEXT) +X +Xsym$(OEXT) : sym.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) sym$(CIEXT) +X +Xtblcmp$(OEXT) : tblcmp.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) tblcmp$(CIEXT) +X +Xyylex$(OEXT) : yylex.c flexdef.h +X $(CCPATH)\$(CC) $(CFLAGS) yylex$(CIEXT) +X +Xflex.man : flex.1 +X nroff -man flex.1 >flex.man +X +Xlint : $(FLEX_C_SOURCES) +X lint $(FLEX_C_SOURCES) > flex.lint +X +Xdistrib : +X mv scan.c initscan.c +X chmod 444 initscan.c +X $(MAKE) $(MFLAGS) clean +X +Xclean : +X rm -f core errs flex *$(OEXT) parse.c *.lint parse.h flex.man tags +X +Xtags : +X ctags $(FLEX_C_SOURCES) +X +Xvms : flex.man +X $(MAKE) $(MFLAGS) distrib +X +Xtest : +X $(FLEX) $(FLEX_FLAGS) scan.l +X $(RENAME) lex_yy.c scan.ctest +X cmp scan.c scan.ctest +X +/ +echo x - Readme2 +sed '/^X/s///' > Readme2 << '/' +XThe changes: (1.7.89 RAL) +X +X - Bug fix: The original flex didn't like trailing spaces in exclusive start +X condition lists ! If you add an trailing space to line 68 in scan.l +X +X "%x SECT2 SECT2PROLOG SECT3 CODEBLOCK PICKUPDEF SC CARETISBOL NUM QUOTE " +X +X you get a misleading error message: +X +X "Syntax error at line 69: bad start condition list" +X +X This bug can either be fixed in parse.y or in scan.l . I have chosen the +X last because there the fix is minimal: Just change the rule (line 128) +X +X "\n" to "{OPTWS}\n" +X +X - Enhancements: +X - new option "-?" that provides some help information about the other +X flags (main.c) +X - new option "-aTMPPATH" that allows a redefinition of the standard +X path for the temporary file (this might be rather large >200k if +X F,f options are selected). (main.c, flexdef.h (l.376)) +X - hexdump of illegal characters -- this proved to be a useful debugging +X tool especialy if invisible control characters occur which weren't +X covered by the rules. (scan.l fprintf statement line 129,...) +X +X - Patches due to TOS +X - General: TOS uses CR,LF as end of line char., Flex wants only a single +X LF as EOL char. Therefore all I/O must be translated using f* calls. +X This is done everywhere besides the YY_INPUT macro (flex.skel (scan.c), +X line 31) that uses a low level 'read'. This should be definitly changed +X to fread, so that all I/O calls occur on the same level. +X - the short action_file_name has been "flexXXXXXX.tmp", but that's too +X much for TOS,MSDOS ! I changed it to "flexXXXX.tmp" in main.c (patch +X covered by the -a option additions) +X - some compilers don't like external names that are ambiguous within +X the first 8 characters. I defined macros that change all these long +X external names to names that are unique within the first 8 characters. +X Just define SHORT_EXTERNAL_NAMES to use this feature (flexdef.h) +X - some statement changes that some compiler don't like: +X typedef *xxx[] -> typedef **xxx (flexdef.h.,l.308) +X "/*" -> '/','*' within a comment in (scan.l, l.209) +X - changed short "lexyy.c" to "lex_yy.c" what's more similar to the unix +X "lex.yy.c" (main.c). +X - a few further really compiler dependent changes provided with +X #ifdef ATARI && LATTICE res. TURBO braces. +X +X - Additional TOS only files +X - Makefile.tos: Common makefile for all TOS compilers. If you add further +X ones please email me the new makefile. +X - flex.lnk: Lattice - GST linker argument extension file +X - flex.tlk: Turbo linker argument extension file +X +X +XAdditional remarks: +X +XI didn't add a new initscan.c (= flex -ist scan.l). The current one is good +Xenough for a first compilation. With this first version of flex you can +Xrebuild your own scan.c and the final flex version ! +X +XMinix ST : +X - I had to "chmem =70000 cv" (>50000) to get flex linked +X - 'memset' (PC 1.3, EFTH40,...) is necessary +X - chmem =90000 flex may be sufficient +X +XMinix PC : +X It should be possible to port Flex to Minix PC. The current sizes of flex +X are: +X Minix ST (ACK) Lattice (TOS) Turbo (TOS) +X +X size 75300 83305 57957 +X compilation time 22' 15' 3'40" +X flex -is scan.l 1'49" 43" 30" +X +X The Minix ST size includes the bad generated code using only a subset of +X the 68000 commands, long addresses only and a huge relocation table. +X Therefore the PC size will be <64 k ! More serious is the fact that I had +X to chmem =90000 flex to get scan.l converted to scan.c . But I never saw +X a more complex lex source than scan.l -- so it should be possible to +X reduce some array sizes without limitation for all day usage. +X +X No one volunteered yet for a Minix PC port -- but if someone will try it +X I would provide him with a new scan.c and some hints. +X +XTOS: +X Don't forget to adapt the flexskel path within flexdef.h ! +X +X +/ +echo x - flex.lnk +sed '/^X/s///' > flex.lnk << '/' +X* +X* +X* linker control file for flex.ttp +X* +X* +X* +XINPUT d:\latt\lib\startup.bin +X* +XINPUT ccl.bin +XINPUT dfa.bin +XINPUT ecs.bin +XINPUT gen.bin +XINPUT misc.bin +XINPUT nfa.bin +XINPUT parse.bin +XINPUT sym.bin +XINPUT tblcmp.bin +XINPUT main.bin +XINPUT yylex.bin +XINPUT scan.bin +X* +XLIBRARY d:\latt\lib\clib.bin +X +/ +echo x - flex.tlk +sed '/^X/s///' > flex.tlk << '/' +Xccl.o +Xdfa.o +Xecs.o +Xgen.o +Xmisc.o +Xnfa.o +Xparse.o +Xsym.o +Xtblcmp.o +Xyylex.o +Xmain.o +Xscan.o +Xd:\turbo\lib\tcstdlib.lib ; standard lib +Xd:\turbo\lib\tcextlib.lib ; extended lib +Xd:\turbo\lib\tctoslib.lib ; TOS lib +Xd:\turbo\lib\tcgemlib.lib ; AES and VDI lib +X-S=200000 +/ diff --git a/MISC/Borland.old/Borland-2.4 b/MISC/Borland.old/Borland-2.4 new file mode 100644 index 0000000..5602a14 --- /dev/null +++ b/MISC/Borland.old/Borland-2.4 @@ -0,0 +1,419 @@ +Received: from 128.140.1.1 by ee.lbl.gov for <vern@ee.lbl.gov> (8.6.9/1.43r) + id HAA01193; Thu, 29 Sep 1994 07:26:54 -0700 +Received: from larry-le0.cc.emory.edu by + emoryu1.cc.emory.edu (5.65/Emory_cc.4.0.1) via SMTP + id AA07292 ; Thu, 29 Sep 94 10:26:41 -0400 +From: tkane01@unix.cc.emory.edu (Terrence O Kane) +Received: by larry.cc.emory.edu (5.0) id AA11757; Thu, 29 Sep 1994 10:26:43 +0500 +Message-Id: <9409291426.AA11757@larry.cc.emory.edu> +Subject: patches and makefile for Borland C 4.02, flex 2.4.7 +To: vern@ee.lbl.gov +Date: Thu, 29 Sep 1994 10:26:42 -0400 (EDT) +X-Mailer: ELM [version 2.4 PL23] +Mime-Version: 1.0 +Content-Type: text/plain; charset=US-ASCII +Content-Transfer-Encoding: 7bit +Content-Length: 9900 + +Enclosed are unified diffs and a makefile for Borland 4.02 + +The changes in the enclosed are 1) make the size parameters for memory +allocation "size_t", 2) change an include file when the lexer is +compiled within 'extern "C" {...}' in a C++ file, and 3) include pragmas +in the header suitable for BCC 4.02 to hush on warnings. + +The latter is done because of the limit on command line size. A tradeoff +exists between putting pragmas in the header, or #defines in the header - +I put in the pragmas since they're suppoed to be ignored unless +understood - *and* they're enclosed in BCC specific ifdefs, anyway. + +All changes are enclosed in "#ifdef __BORLANDC__". + + + + + +--- misc.c Tue Jan 04 14:33:10 1994 ++++ ../misc.c Wed Sep 28 18:44:32 1994 +@@ -55,15 +55,19 @@ + action_index += len; + } + + + /* allocate_array - allocate memory for an integer array of the given size */ + + void *allocate_array( size, element_size ) ++#ifndef __BORLANDC__ + int size, element_size; ++#else /* __BORLANDC__ */ ++size_t size, element_size; ++#endif /* __BORLANDC__ */ + { + register void *mem; + + /* On 16-bit int machines (e.g., 80286) we might be trying to + * allocate more than a signed int can hold, and that won't + * work. Cheap test: + */ +@@ -634,15 +638,19 @@ + } + + + /* reallocate_array - increase the size of a dynamic array */ + + void *reallocate_array( array, size, element_size ) + void *array; ++#ifndef __BORLANDC__ + int size, element_size; ++#else /* __BORLANDC__ */ ++size_t size, element_size; ++#endif /* __BORLANDC__ */ + { + register void *new_array; + + /* Same worry as in allocate_array(): */ + if ( size * element_size <= 0 ) + flexfatal( + "attempt to increase array size by less than 1 byte" ); +@@ -739,15 +747,19 @@ + } + + + /* The following is only needed when building flex's parser using certain + * broken versions of bison. + */ + void *yy_flex_xmalloc( size ) ++#ifndef __BORLANDC__ + int size; ++#else /* __BORLANDC__ */ ++size_t size; ++#endif /* __BORLANDC__ */ + { + void *result = flex_alloc( size ); + + if ( ! result ) + flexfatal( "memory allocation failed in yy_flex_xmalloc()" ); + + return result; + + + + + +--- skel.c Wed Aug 03 11:38:32 1994 ++++ ../skel.c Wed Sep 28 18:50:58 1994 +@@ -26,15 +26,19 @@ + "", + "#ifdef __cplusplus", + "", + "#include <stdlib.h>", + "%+", + "class istream;", + "%*", ++ "#ifndef __BORLANDC__", + "#include <unistd.h>", ++ "#else /* __BORLANDC__ */", ++ "#include <io.h>", ++ "#endif /* __BORLANDC__ */", + "", + "/* Use prototypes in function declarations. */", + "#define YY_USE_PROTOS", + "", + "/* The \"const\" storage-class-modifier is valid. */", + "#define YY_USE_CONST", + "", +@@ -240,16 +244,21 @@ + "static int yy_start_stack_depth = 0;", + "static int *yy_start_stack = 0;", + "static void yy_push_state YY_PROTO(( int new_state ));", + "static void yy_pop_state YY_PROTO(( void ));", + "static int yy_top_state YY_PROTO(( void ));", + "%*", + "", ++ "#ifndef __BORLANDC__", + "static void *yy_flex_alloc YY_PROTO(( unsigned int ));", + "static void *yy_flex_realloc YY_PROTO(( void *, unsigned int ));", ++ "#else /* __BORLANDC__ */", ++ "static void *yy_flex_alloc YY_PROTO(( size_t ));", ++ "static void *yy_flex_realloc YY_PROTO(( void *, size_t ));", ++ "#endif /* __BORLANDC__ */", + "static void yy_flex_free YY_PROTO(( void * ));", + "", + "#define yy_new_buffer yy_create_buffer", + "", + "%% yytext/yyin/yyout/yy_state_type/yylineno etc. def's & init go here", + "", + "#ifndef yytext_ptr", + + + + + +--- initscan.c Wed Aug 03 11:42:46 1994 ++++ ../initscan.c Wed Sep 28 18:51:34 1994 +@@ -16,15 +16,19 @@ + #endif + #endif + + + #ifdef __cplusplus + + #include <stdlib.h> ++#ifndef __BORLANDC__ + #include <unistd.h> ++#else /* __BORLANDC__ */ ++#include <io.h> ++#endif /* __BORLANDC__ */ + + /* Use prototypes in function declarations. */ + #define YY_USE_PROTOS + + /* The "const" storage-class-modifier is valid. */ + #define YY_USE_CONST + +@@ -220,16 +224,21 @@ + static int yy_start_stack_ptr = 0; + static int yy_start_stack_depth = 0; + static int *yy_start_stack = 0; + static void yy_push_state YY_PROTO(( int new_state )); + static void yy_pop_state YY_PROTO(( void )); + static int yy_top_state YY_PROTO(( void )); + ++#ifndef __BORLANDC__ + static void *yy_flex_alloc YY_PROTO(( unsigned int )); + static void *yy_flex_realloc YY_PROTO(( void *, unsigned int )); ++#else /* __BORLANDC__ */ ++static void *yy_flex_alloc YY_PROTO(( size_t )); ++static void *yy_flex_realloc YY_PROTO(( void *, size_t )); ++#endif /* __BORLANDC__ */ + static void yy_flex_free YY_PROTO(( void * )); + + #define yy_new_buffer yy_create_buffer + + #define INITIAL 0 + #define SECT2 1 + #define SECT2PROLOG 2 + + + + + +--- flexdef.h Tue Jan 04 14:33:14 1994 ++++ ../flexdef.h Wed Sep 28 18:53:44 1994 +@@ -27,14 +27,25 @@ + */ + + /* @(#) $Header: flexdef.h,v 1.2 94/01/04 14:33:14 vern Exp $ (LBL) */ + + #include <stdio.h> + #include <ctype.h> + ++#ifdef __BORLANDC__ ++#include <malloc.h> ++ ++#pragma warn -pro ++#pragma warn -rch ++#pragma warn -use ++#pragma warn -aus ++#pragma warn -par ++#pragma warn -pia ++ ++#endif /* __BORLANDC__ */ + #if HAVE_STRING_H + #include <string.h> + #else + #include <strings.h> + #endif + + #if __STDC__ +@@ -607,19 +618,29 @@ + */ + + extern char nmstr[MAXLINE]; + extern int sectnum, nummt, hshcol, dfaeql, numeps, eps2, num_reallocs; + extern int tmpuses, totnst, peakpairs, numuniq, numdup, hshsave; + extern int num_backing_up, bol_needed; + ++#ifndef __BORLANDC__ + void *allocate_array PROTO((int, int)); + void *reallocate_array PROTO((void*, int, int)); ++#else /* __BORLANDC__ */ ++void *allocate_array PROTO((size_t, size_t)); ++void *reallocate_array PROTO((void*, size_t, size_t)); ++#endif /* __BORLANDC__ */ + ++#ifndef __BORLANDC__ + void *flex_alloc PROTO((unsigned int)); + void *flex_realloc PROTO((void*, unsigned int)); ++#else /* __BORLANDC__ */ ++void *flex_alloc PROTO((size_t)); ++void *flex_realloc PROTO((void*, size_t)); ++#endif /* __BORLANDC__ */ + void flex_free PROTO((void*)); + + #define allocate_integer_array(size) \ + (int *) allocate_array( size, sizeof( int ) ) + + #define reallocate_integer_array(array,size) \ + (int *) reallocate_array( (void *) array, size, sizeof( int ) ) +@@ -772,15 +793,19 @@ + /* Write out one section of the skeleton file. */ + extern void skelout PROTO((void)); + + /* Output a yy_trans_info structure. */ + extern void transition_struct_out PROTO((int, int)); + + /* Only needed when using certain broken versions of bison to build parse.c. */ ++#ifndef __BORLANDC__ + extern void *yy_flex_xmalloc PROTO(( int )); ++#else /* __BORLANDC__ */ ++extern void *yy_flex_xmalloc PROTO(( size_t )); ++#endif /* __BORLANDC__ */ + + /* Set a region of memory to 0. */ + extern void zero_out PROTO((char *, int)); + + + /* from file nfa.c */ + + + + + +############################################################################### +# Makefile for flex 2.4.7 with Borland C/C++ version 4.02 +# +# This will probably need to be adjusted for your existing lexer/parser +# generators. See definitions for FLEX and YACC near the bottom of the +# makefile. +# +# Copy initscan.c to scan.c to make your first executable. After that, +# you may choose to try alternate compression options for your everyday +# flex executable. +# +# This will build flex with the large model. Don't use huge, but if you +# feel like experimenting with other models, post your success stories to +# comp.compilers, OK? +# +# This makefile does *not* implement the big testing found in "makefile.in". +# +# I also assume the availability of sed and the gnu file utilities on the +# system - they're readily available, so if you don't have them, why not? +# <grin> +# +# The resulting generated lexer (the real goal, right?) will compile +# (and run nicely, too) as a .c file, as well as being included such as +# extern "C" { #include "lexyyc" } in a .cplusplus file. +# +############################################################################### + +DEBUG = 1 + +.autodepend + +all: flex.exe + +############################################################################### +# +# standard utilitities? ha. +# + +CC = bcc +CPP = bcc + +############################################################################### +# + +MODEL = l + +!if $(DEBUG) == 1 +!message Building with debug. +debugCompile = -v +debugLink = /v +!else +!message Building without debug. +debugCompile = +debugLink = +!endif + +LOADER = c0$(MODEL).obj +LIBS = c$(MODEL).lib +LINKFLAGS = $(debugLink) + +DATASEG = -dc -Ff +SizeOPT = -Os -G- +Defines = -DSHORT_FILE_NAMES=1 -DHAVE_STRING_H=1 + +COMMON = -A -c -m$(MODEL) $(SizeOPT) $(DATASEG) $(Defines) $(debugCompile) +CFLAGS = -o$@ $(COMMON) +CCFLAGS = -o$@ $(COMMON) -Pcc + +############################################################################### + +.SUFFIXES: .cc + +.cc.obj: + $(CPP) $(CCFLAGS) $< + +.c.obj: + $(CPP) $(CFLAGS) $< + +############################################################################### +# +# source & object files +# + +SRC = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.c \ + scan.c sym.c tblcmp.c yylex.c skel.c + +OBJS = $(SRC:.c=.obj) + +objects: $(OBJS) + @echo $(OBJS) + +############################################################################### +# +# Executable +# + +flex.exe: $(OBJS) + tlink $(LINKFLAGS) @&&! +$(LOADER) $** +$&.exe +$&.map +$(LIBS) +! + +# +############################################################################### +# +# Lex files +# + +FLEX = .\flex +FLEX_FLAGS = -ist + +scan.c: scan.l + $(FLEX) $(FLEX_FLAGS) scan.l >scan.tmp + sed s,\"$(srcdir)/scan.l\",\"scan.l\", <scan.tmp >scan.c + @rm scan.tmp + +############################################################################### +# +# YACC files +# + +YACC = .\bison +YFLAGS = -vdyl + +parse.c: parse.y + $(YACC) -ydl parse.y + @sed "/extern char.*malloc/d" <y_tab.c >parse.c + @rm -f y_tab.c + @mv y_tab.h parse.h + +# +# end Makefile +# +############################################################################### + diff --git a/MISC/Borland.old/Turbo-C b/MISC/Borland.old/Turbo-C new file mode 100644 index 0000000..bfe8a92 --- /dev/null +++ b/MISC/Borland.old/Turbo-C @@ -0,0 +1,179 @@ +Received: from 128.84.254.220 by ee.lbl.gov for <vern@ee.lbl.gov> (8.6.8.1/1.43r) + id PAA27266; Mon, 18 Apr 1994 15:08:26 -0700 +Received: from CLOYD.CS.CORNELL.EDU by thialfi.cs.cornell.edu (5.67/I-1.99E) + id AA28742; Mon, 18 Apr 94 18:08:14 -0400 +Received: from iraun1.ira.uka.de by cloyd.cs.cornell.edu (5.67/I-1.99D) + id AA19613; Mon, 18 Apr 94 18:08:19 -0400 +Received: from t500i2.telematik.informatik. (actually t500i2.telematik.informatik.uni-karlsruhe.de) + by iraun1.ira.uka.de with SMTP (PP); Tue, 19 Apr 1994 00:07:55 +0200 +Received: by t500i2.telematik.informatik.uni-karlsruhe.de (5.57/Ultrix3.0-C) + id AA10269; Tue, 19 Apr 94 00:09:14 +0200 +From: beigl@t500i2.telematik.informatik.uni-karlsruhe.de (Michael Beigl) +Message-Id: <9404182209.AA10269@t500i2.telematik.informatik.uni-karlsruhe.de> +Subject: Makefile-TurboC +To: vern@cs.cornell.edu +Date: Tue, 19 Apr 1994 00:09:13 +0200 (MET DST) +X-Mailer: ELM [version 2.4 PL22] +Mime-Version: 1.0 +Content-Type: text/plain; charset=US-ASCII +Content-Transfer-Encoding: 7bit +Content-Length: 2739 + +Hello + +Here are some additional adjustments to my Makefile. I was using "pure" DOS +and an old Turbo C++ 1.0 version, so I had some problems with systemtools +like mv etc. and problems with variables in my Makefile. + +Now follows my Makefile +############################################################################ +# make file for "flex" tool + +# @(#) $Header: Makefile,v 2.3 89/06/20 17:27:12 vern Exp $ (LBL) + +# Porting considerations: +# +# For BSD machines: +# CFLAGS = +# LDFLAGS = -s +# LINK = $(CC) $(CFLAGS) -o flex $(LDFLAGS) $(FLEXOBJS) +# SKELETON_DIR = . +# SKELETON_FILE = flex.skel +# SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_DIR)/$(SKELETON_FILE)\" +# O = o +# YTAB = y.tab +# FLEX = ./flex +# +# For System V Unix or Vax/VMS machines, merely add: +# CFLAGS = -DSYS_V +# +# For MS-DOS, Turbo C: +CC = tcc +# -2+ 286 Options +CFLAGS = -DSYS_V -DMS_DOS -O -G -Z -ml -v -2 +# /3 enable 32 bit processing +# /ye expanded memory swapping +# /yx extended memory swapping +LINK = tlink @flex.lnk/c/x/v/3/ye +SKELETON_DIR = . +SKELETON_FILE = flex.skl +SKELFLAGS = -DDEFAULT_SKELETON_FILE="$(SKELETON_DIR)/$(SKELETON_FILE)" +O = obj +EXE = .exe +YTAB = y_tab +FLEX = flex +YACC = /yacc/byacc + +# +# the first time around use "make first_flex" +# + +FLEX_FLAGS = + +FLEXOBJS = \ + ccl.$O \ + dfa.$O \ + ecs.$O \ + gen.$O \ + main.$O \ + misc.$O \ + nfa.$O \ + parse.$O \ + scan.$O \ + sym.$O \ + tblcmp.$O \ + yylex.$O + +FLEX_C_SOURCES = \ + ccl.c \ + dfa.c \ + ecs.c \ + gen.c \ + main.c \ + misc.c \ + nfa.c \ + parse.c \ + scan.c \ + sym.c \ + tblcmp.c \ + yylex.c + +FLEX_C_SOURCES_1 = \ + ccl.c \ + dfa.c \ + ecs.c \ + gen.c \ + main.c \ + misc.c + +FLEX_C_SOURCES_2 = \ + nfa.c \ + parse.c \ + scan.c \ + sym.c \ + tblcmp.c \ + yylex.c + +flex.exe: $(FLEXOBJS) + $(LINK) + + +flex: $(FLEX_C_SOURCES) + $(CC) $(CFLAGS) -c $(SKELFLAGS) $(FLEX_C_SOURCES_1) + $(CC) $(CFLAGS) -c $(SKELFLAGS) $(FLEX_C_SOURCES_2) + $(LINK) + + +first_flex: + copy initscan.c scan.c + $(MAKE) flex + +parse.h parse.c: parse.y + $(YACC) -d parse.y + @rename $(YTAB).c parse.c + @rename $(YTAB).h parse.h + +scan.c: scan.l + $(FLEX) -ist $(FLEX_FLAGS) scan.l >scan.c + +scan.$O: scan.c parse.h + +main.$O: main.c + $(CC) $(CFLAGS) -c $(SKELFLAGS) main.c + +flex.man: flex.1 + nroff -man flex.1 >flex.man + +lint: $(FLEX_C_SOURCES) + lint $(FLEX_C_SOURCES) > flex.lint + +distrib: + rename scan.c initscan.c + attrib +R -A -H -S initscan.c + $(MAKE) clean + +clean: + del *.obj + del *.lint + del core + del errs + del flex.exe + del parse.c + del parse.h + del flex.man + del tags + +tags: + ctags $(FLEX_C_SOURCES) + +vms: flex.man + $(MAKE) distrib + +test: + $(FLEX) -ist $(FLEX_FLAGS) scan.l | diff scan.c - + +############################################################################ + +I think this Makefile will help some other simple DOS user + + M. Beigl diff --git a/MISC/Borland/Makefile b/MISC/Borland/Makefile new file mode 100644 index 0000000..3b20090 --- /dev/null +++ b/MISC/Borland/Makefile @@ -0,0 +1,163 @@ +############################################################################### +# Makefile for flex 2.5.0.6 (beta) with Borland C/C++ version 4.02 +# +# This will probably need to be adjusted for your existing lexer/parser +# generators. See definitions for FLEX and YACC near the bottom of the +# makefile. +# +# This makefile builds initflex.exe and flex.exe by default. It +# removes initflex.exe after making flex.exe. After that, you may +# choose to try alternate compression options for your everyday flex +# executable. +# +# This will build flex with the large model. Don't use huge, but if you +# feel like experimenting with other models, post your success stories to +# comp.compilers, OK? +# +# This makefile does *not* implement the big testing found in "makefile.in". +# +# I also assume the availability of sed and the gnu file utilities on the +# system - they're readily available, so if you don't have them, why not? +# <grin> +# +# The resulting generated lexer (the real goal, right?) will compile +# (and run nicely, too) as a .c file, as well as being included such as +# extern "C" { #include "lexyyc" } in a .cplusplus file. +# +############################################################################### + +DEBUG = 1 + +.autodepend + +all: initflex.exe flex.exe + rm initflex.exe initflex.map + +############################################################################### +# +# standard utilitities? ha. +# + +CC = bcc +CPP = bcc + +############################################################################### +# + +MODEL = l + +!if $(DEBUG) == 1 +!message Building with debug. +debugCompile = -v +debugLink = /v +!else +!message Building without debug. +debugCompile = +debugLink = +!endif + +LOADER = c0$(MODEL).obj +LIBS = c$(MODEL).lib +LINKFLAGS = $(debugLink) + +DATASEG = -dc -Ff +SizeOPT = -Os -G- +Defines = + +COMMON = -A -c -m$(MODEL) $(SizeOPT) $(DATASEG) $(Defines) $(debugCompile) +CFLAGS = -o$@ $(COMMON) +CCFLAGS = -o$@ $(COMMON) -Pcc + +############################################################################### + +.SUFFIXES: .cc + +.cc.obj: + $(CPP) $(CCFLAGS) $< + +.c.obj: + $(CPP) $(CFLAGS) $< + +############################################################################### +# +# source & object files +# + +BASESRC = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.c \ + sym.c tblcmp.c yylex.c skel.c + +INITSRC = $(BASESRC) initscan.c + +INITOBJS = $(INITSRC:.c=.obj) + +SRC = $(BASESRC) scan.c + +OBJS = $(SRC:.c=.obj) + +objects: $(OBJS) + @echo $(OBJS) + +############################################################################### +# +# Executable +# + +initflex.exe: $(INITOBJS) + tlink $(LINKFLAGS) @&&! +$(LOADER) $** +$&.exe + +$(LIBS) +! + +flex.exe: $(OBJS) + tlink $(LINKFLAGS) @&&! +$(LOADER) $** +$&.exe + +$(LIBS) +! + +# +############################################################################### +# +# Lex files +# + +FLEX = .\initflex +FLEX_FLAGS = -ist + +scan.c: scan.l + $(FLEX) $(FLEX_FLAGS) scan.l >scan.tmp + sed s,\"$(srcdir)/scan.l\",\"scan.l\", <scan.tmp >scan.c + @rm scan.tmp + +############################################################################### +# +# YACC files +# + +YACC = .\bison +YFLAGS = -vdyl + +parse.c: parse.y + $(YACC) -ydl parse.y + @sed "/extern char.*malloc/d" <y_tab.c >parse.c + @rm -f y_tab.c + @mv y_tab.h parse.h + +############################################################################### +# +# cleanup +# + +clean: + -rm *.obj *.map initflex.exe + +realclean: clean + -rm flex.exe + +# +# end Makefile +# +############################################################################### diff --git a/MISC/Borland/NOTES b/MISC/Borland/NOTES new file mode 100644 index 0000000..caac3cc --- /dev/null +++ b/MISC/Borland/NOTES @@ -0,0 +1,66 @@ +Received: from 128.140.1.1 by ee.lbl.gov for <vern@ee.lbl.gov> (8.6.9/1.43r) + id PAA03966; Tue, 24 Jan 1995 15:03:57 -0800 +Received: from larry-le0.cc.emory.edu by + emoryu1.cc.emory.edu (5.65/Emory_cc.4.0.1) via SMTP + id AA24158 ; Tue, 24 Jan 95 17:18:18 -0500 +From: tkane01@unix.cc.emory.edu (Terrence O Kane) +Received: by larry.cc.emory.edu (5.0) id AA21979; Tue, 24 Jan 1995 17:17:40 -0500 +Message-Id: <9501242217.AA21979@larry.cc.emory.edu> +Subject: Re: Beta test for DOS +To: vern@ee.lbl.gov (Vern Paxson) +Date: Tue, 24 Jan 1995 17:17:38 -0500 (EST) +In-Reply-To: <199501232138.NAA11430@daffy.ee.lbl.gov> from "Vern Paxson" at Jan 23, 95 01:38:02 pm +X-Mailer: ELM [version 2.4 PL23] +Mime-Version: 1.0 +Content-Type: text/plain; charset=US-ASCII +Content-Transfer-Encoding: 7bit +Content-Length: 5792 + +Vern, + +I've made flex 2.5.0.6 successfully with no changes to the source +code at all. I'm including the Borland 4.02 makefile and config.h +at the end of this message. + +When you're ready for release, I'll be happy to take care of getting +the appropriate archive files up to Simtel if you wish. + +I have not used this version for any of my "work-related" scanners +yet, but have run the fastwc benchmark. The compiles were for large +memory model and optimization for fastest possible code. The test +machine was a Pentium-90 (hey! timing output was integer!) with +enhanced IDE on a PCI bus and no file system caching. I ran the +test on two different input files. + +(Times are in seconds.) + +The first input was a typical postscript file concatenated 10 times; +by typical, I mean that there were relatively few spaces, lots of lines +with no space, using lots of parentheses. + + lines words characters + 91200 356260 5889240 + + mywc 8.89 + wc1s 15.22 default + wc1 10.82 -Cf + wc2 10.16 -Cf + wc3 9.17 -Cf + wc4 9.22 -Cf + wc5 10.98 -Cf + +The second test run was on a file that consisted of 20 concatenations +of 'misc/flex.man'. + + lines words characters + 69960 305140 2399960 + + mywc 4.01 + wc1s 6.87 + wc1 5.32 + wc2 4.95 + wc3 4.12 + wc4 4.12 + wc5 5.05 + +[[Makefile and config.h moved to separate files -VP]] diff --git a/MISC/Borland/config.h b/MISC/Borland/config.h new file mode 100644 index 0000000..5e210da --- /dev/null +++ b/MISC/Borland/config.h @@ -0,0 +1,29 @@ +/* $Header: /home/daffy/u0/vern/flex/RCS/conf.in,v 1.2 95/01/09 12:11:51 vern Exp $ */ +/* ------------------------------------------------ */ +/* version of config.h for Borland C/C++ v4.02 */ +/* flex version 2.5.0.6 (beta) */ +/* ------------------------------------------------ */ + +/* Define to empty if the keyword does not work. */ +#undef const + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +#undef size_t + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define if you have the <malloc.h> header file. */ +#define HAVE_MALLOC_H 1 + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define if you have <alloca.h> and it should be used (not on Ultrix). */ +#undef HAVE_ALLOCA_H + +/* Define if platform-specific command line handling is necessary. */ +#undef NEED_ARGV_FIXUP diff --git a/MISC/EBCDIC b/MISC/EBCDIC new file mode 100644 index 0000000..e4ebd71 --- /dev/null +++ b/MISC/EBCDIC @@ -0,0 +1,48 @@ +Return-Path: Mark_Ganter@liz.com +Received: from 192.216.61.11 by horse.ee.lbl.gov for vern (5.65/1.43r) + id AA02152; Tue, 11 Jan 94 06:19:36 -0800 +Received: from melonville.radiomail.net (mayberry.radiomail.net) by radiomail.net with SMTP id AA20220 + (5.65c+/IDA-1.4.4 for <vern@horse.ee.lbl.gov>); Tue, 11 Jan 1994 06:19:35 -0800 +Message-Id: <199401111419.AA20220@radiomail.net> +Received: from liz.com by melonville.radiomail.net with CCGW-1.7(930217); + Tue, 11 Jan 94 06:19:17 +From: Mark_Ganter@liz.com +Date: 11 Jan 94 09:05 +To: vern@horse.ee.lbl.gov +Subject: Re[2]: Flex on an AS400 + + I have no problem at all with being a contact for advice + on porting Flex to EBCDIC. The code was put on an AS400, so + filing the message under /MISC/AS400 is more appropriate. + The port was an interesting, very educational experience. + Thanks again. + + + Mark + +------------------------------------------------------------------------ +Date: Tue, 18 Apr 1995 12:33:48 -0400 +To: "Michael W. Duffy" <mduffy@netcom.com>, Mark_Ganter@liz.com, + vern@ee.lbl.gov (Vern Paxson), slayten@cas.org +From: slayten@cas.org (Steve Layten) +Subject: RE: Porting LEX scanner on EBCDIC machine +X-Mailer: <Windows Eudora Version 2.0.2> +Content-Length: 918 + +I came in late on this discussion - I don't follow comp.compilers very +closely these days because my job has shifted somewhat. + +I ported (quite some time ago) flex 2.3.6 to an IBM 3090 under MVS, using +the SAS C compiler. The approach I used was, as Vern suggested, was to +translate the tables to reflect EBCDIC-based indices. I haven't even +downloaded flex 2.4 yet, so I don't know what's in the current distribution. +My patches were in the flex 2.3 distribution for a while in the MISC +directory. If you want any more info feel free to drop me a line - I still +have (somewhere) the patches that I created from version 2.3.6. + +Steve Layten +-- +Steven W. Layten, Senior Research Scientist +Chemical Abstracts Service PO BOX 3012, Columbus, OH 43210 +1 614 447 3600 +INET: slayten@cas.org FAX: +1 614 447 3813 +# # Speaking only for myself, and NOT for Chemical Abstracts Service! # # # diff --git a/MISC/MSDOS/MSC70.make b/MISC/MSDOS/MSC70.make new file mode 100644 index 0000000..5980f3a --- /dev/null +++ b/MISC/MSDOS/MSC70.make @@ -0,0 +1,115 @@ +# +# make file for "flex" tool +# @(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/Makefile,v 2.9 90/05/26 17:28:44 + vern Exp $ (LBL) +# +# the first time around use "make f_flex" +# +# This makefile is specific for Microsoft's C/C++ compiler (v7), nmake and +# lib +# - Paul Stuart, Jan 93 (pjs@scammell.ecos.tne.oz.au) +# + + +SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"c:/src/flex/flex.skl\" +CFLAGS = -nologo -AL -W2 -F 8000 -Ox -Gt16000 -DMS_DOS -DUSG +LDFLAGS = /nologo /NOI /BATCH /ONERROR:NOEXE /STACK:8000 +FLEX_FLAGS = -ist8 -Sflex.skl + +FLEX = .\flex.exe +CC = cl +YACC = c:\lib\byacc +MAKE = nmake /nologo + +# +# break obj-list into two because of 128 character command-line limit of +# Microsoft's link and lib utilities. +# +FLEXOBJS1 = \ + ccl.obj \ + dfa.obj \ + ecs.obj \ + gen.obj \ + main.obj \ + misc.obj \ + nfa.obj \ + parse.obj + +FLEXOBJS2 = \ + scan.obj \ + sym.obj \ + tblcmp.obj \ + yylex.obj + +FLEX_C_SOURCES = \ + ccl.c \ + dfa.c \ + ecs.c \ + gen.c \ + main.c \ + misc.c \ + nfa.c \ + parse.c \ + scan.c \ + sym.c \ + tblcmp.c \ + yylex.c + +FLEX_LIB_OBJS = \ + libmain.obj + + +all : flex.exe + +# +# lib is used to get around the 128 character command-line limit of 'link'. +# +flex.exe : $(FLEXOBJS1) $(FLEXOBJS2) + lib /nologo tmplib $(FLEXOBJS1); + link $(LDFLAGS) $(FLEXOBJS2),$*.exe,,tmplib; + del tmplib.lib + +f_flex: + copy initscan.c scan.c + touch scan.c + @echo compiling first flex + $(MAKE) flex.exe + del scan.c + @echo using first flex to generate final version... + $(MAKE) flex.exe + +# +# general inference rule +# +.c.obj: + $(CC) -c $(CFLAGS) $*.c + +parse.h parse.c : parse.y + $(YACC) -d parse.y + @mv y_tab.c parse.c + @mv y_tab.h parse.h + +scan.c : scan.l + $(FLEX) $(FLEX_FLAGS) $(COMPRESSION) scan.l >scan.c + + +scan.obj : scan.c parse.h flexdef.h + +main.obj : main.c flexdef.h + $(CC) $(CFLAGS) -c $(SKELFLAGS) main.c + +ccl.obj : ccl.c flexdef.h +dfa.obj : dfa.c flexdef.h +ecs.obj : ecs.c flexdef.h +gen.obj : gen.c flexdef.h +misc.obj : misc.c flexdef.h +nfa.obj : nfa.c flexdef.h +parse.obj : parse.c flexdef.h +sym.obj : sym.c flexdef.h +tblcmp.obj : tblcmp.c flexdef.h +yylex.obj : yylex.c flexdef.h + + +clean : + del *.obj + del *.map diff --git a/MISC/MSDOS/configur.bat b/MISC/MSDOS/configur.bat new file mode 100644 index 0000000..dbea0e4 --- /dev/null +++ b/MISC/MSDOS/configur.bat @@ -0,0 +1,6 @@ +@echo off + +sed -e "s/y\.tab\./parse_tab\./" -e "/sed/ s/'/\"/g" < Makefile.in > Makefile +sed -f MISC/MSDOS/djgpp.sed Makefile.in > Makefile + +update initscan.c scan.c diff --git a/MISC/MSDOS/djgpp.sed b/MISC/MSDOS/djgpp.sed new file mode 100644 index 0000000..b436113 --- /dev/null +++ b/MISC/MSDOS/djgpp.sed @@ -0,0 +1,12 @@ +s/y\.tab\./parse_tab\./ +s/@DEFS@/-DMS_DOS/ +s/@LIBS@// +s/@srcdir@/./ +s/@YACC@/bison/ +s/@CC@/gcc/ +s/@RANLIB@/ranlib/ +s/@ALLOCA@// +/^flex/ s/\.bootstrap// +/sed.*extern.*malloc/ c\ + @mv parse_tab.c parse.c +/rm.*parse_tab.c/ d diff --git a/MISC/MVS/MVS.mail b/MISC/MVS/MVS.mail new file mode 100644 index 0000000..5a75e5b --- /dev/null +++ b/MISC/MVS/MVS.mail @@ -0,0 +1,56 @@ +(Message inbox:40) +Date: Tue, 17 Jul 1990 11:56 EDT +From: swl26%CAS.BITNET@CORNELLC.cit.cornell.edu +Subject: Re(2): port of flex-2.3 to IBM/MVS +To: vern@cs.cornell.edu + +>Message received. I'm sending this response to +>swl26%CAS.BITNET@CORNELLC.cit.cornell.edu. + +Great -- we can talk. + +>How extensive are the diffs? If they're fairly short then yes, please +>send them. If they're pretty involved then probably not, as I'm not aware +>of many MVS flex users ... + +I've built a context diff file which is ~850 lines. + +Summary of major changes necessary: + + o a new initscan.c -- because MVS is ebcdic, the initial scanner had + to be changed. I built a scanner from scan.l using -Ce, then + hand-manipulated the yy_ec table to fit EBCDIC instead of ASCII + chars. (This is not included in the diff file above.) + + o changes in main and flexdef.h to change how files are handled. (No + dynamic file names, etc.) + + o Some lines had to be shortened to 80 bytes. This mostly impacted + your RCSID lines, which with the full path name were too long. + + o SASC and the linker don't allow externals to be longer than 8 chars. + I thus wrote a Q&D program to shorten all externals. (This would be + a separate file 'fixit.l', which is not included in the diffs.) + + o There are several places where there are tests for 'isascii' (which I + deleted conditionally, using #ifdefs), and tests for 'A' <= x <= 'Z' + which I changed to 'isupper'. + +Many of the changes I've incorporated without impacting other systems. +Others are with 'ifdefs'. Still others (the short external names and new +initscan) are 'isolable' as separate files which would have to be +included with a distribution. + +Obviously, you are not going to want to (even if you can :-) ) worry about +testing new versions in the MVS environment. Likewise, I don't have the +time or resources to provide much support. (I'm not sure my management +would allow any :-( ) + +With all of the above feel free to say "Thanks, but no thanks." If you +want to see the diffs anyway, I will certainly mail them. + +Thanks for your time and efforts. + +Steve Layten +Chemical Abstracts Service, PO Box 3012, Columbus, OH 43210, +1 614 421-3600 +INET: swl26%cas.BITNET@CUNYVM.CUNY.Edu diff --git a/MISC/MVS/MVS.todo b/MISC/MVS/MVS.todo new file mode 100644 index 0000000..b6060ec --- /dev/null +++ b/MISC/MVS/MVS.todo @@ -0,0 +1,23 @@ +(Message inbox:47) +Date: Wed, 18 Jul 1990 14:16 EDT +From: swl26%CAS.BITNET@CORNELLC.cit.cornell.edu +Subject: Re(2): Re(2): diffs for mvs port of flex-2.3 +To: vern@cs.cornell.edu + + + +>Thanks, I've successfully unpacked the archive. I may simply distribute +>the diffs as a set of files in the MISC/ directory rather than incorporating +>them into the 2.4 release. Let me know if you don't want me to do so. +> +> Vern + + +Thank you! What you propose is okay. I might suggest, however, that you +look in the diffs for dfa.c, misc.c, and yylex.c, and consider removing +some of the ASCIIisms which are probably unnecessary. The manner in +which I made the changes was pure brute force, and there might be a +better way, but the changes I made there shouldn't hurt anything. + + regards, + Steve diff --git a/MISC/MVS/MVS.uudecode b/MISC/MVS/MVS.uudecode new file mode 100644 index 0000000..2b8e802 --- /dev/null +++ b/MISC/MVS/MVS.uudecode @@ -0,0 +1,341 @@ +Received: from CU-ARPA.CS.CORNELL.EDU by loki.cs.cornell.edu (5.61/I-1.91f) + id AA25874; Wed, 18 Jul 90 12:02:22 -0400 +Message-Id: <9007181320.AA24810@cu-arpa.cs.cornell.edu> +Received: from CORNELLC.CIT.CORNELL.EDU by cu-arpa.cs.cornell.edu (5.61+2/1.91d) + id AA24810; Wed, 18 Jul 90 09:20:21 -0400 +Received: from CORNELLC by CORNELLC.cit.cornell.edu (IBM VM SMTP R1.2.1MX) with BSMTP id 6769; Wed, 18 Jul 90 09:18:46 EDT +Received: from CAS.BITNET (MAILER) by CORNELLC (Mailer R2.05X) with BSMTP id + 5378; Wed, 18 Jul 90 09:18:38 EDT +From: swl26%CAS.BITNET@CORNELLC.cit.cornell.edu +Date: Wed, 18 Jul 1990 09:16 EDT +Subject: Re(2): diffs for mvs port of flex-2.3 +In-Reply-To: Your message of Tue, 17 Jul 90 17:42:3 +To: vern@cs.cornell.edu + +Sorry about the trailing blank problem. It's farily common with data sent +through bitnet paths, but ever the optimist ... + +>I think there should be an 'M' at the beginning of the second line. + +This isn't a problem. I believe that the first byte of the line indicates +it's length (in some manner). + +Rather than re-send the data, how about a uudecode that compensates for +the trailing blank problem? I manually mangled the uuencoded file and ran +the following decode, and it seemed to work. + +#! /bin/sh +# This is a shell archive. Remove anything before this line, then feed it +# into a shell via "sh file" or similar. To overwrite existing files, +# type "sh file -c". +# The tool that generated this appeared in the comp.sources.unix newsgroup; +# send mail to comp-sources-unix@uunet.uu.net if you want that tool. +# If this archive is complete, you will see the following message at the end: +# "End of shell archive." +# Contents: uudecode.c +# Wrapped by swl26@swl26aws on Wed Jul 18 08:59:24 1990 +PATH=/bin:/usr/bin:/usr/ucb ; export PATH +if test -f 'uudecode.c' -a "${1}" != "-c" ; then + echo shar: Will not clobber existing file \"'uudecode.c'\" +else +echo shar: Extracting \"'uudecode.c'\" \(6418 characters\) +sed "s/^X//" >'uudecode.c' <<'END_OF_FILE' +X/* #ifndef lint +Xstatic char sccsid[] = "@(#)uudecode.c 5.3-1 (Berkeley) 9/1/87"; +X#endif */ +X +X/* Written by Mark Horton */ +X/* Modified by ajr (Alan J Rosenthatl,flaps@utcsri.UUCP) to use checksums */ +X/* Modified by fnf (Fred Fish,well!fnf) to use Keith Pyle's suggestion for +X compatibility */ +X/* Modified by bcn (Bryce Nesbitt,ucbvax!cogsci!bryce) to fix a misleading +X error message on the Amiga port, to fix a bug that prevented decoding +X certain files, to work even if trailing spaces have been removed from a +X file, to check the filesize (if present), to add some error checking, to +X loop for multiple decodes from a single file, and to handle common +X BITNET mangling. Also kludged around a missing string function in Aztec +X C */ +X +X/* +X * uudecode [input] +X * +X * Decode a file encoded with uuencode. WIll extract multiple encoded +X * modules from a single file. Can deal with most mangled files, including +X * BITNET. +X */ +X +X#include <stdio.h> +X#include <ctype.h> +X +X#ifdef AMIGA +X#define AMIGA_LATTICE /* Set for Amiga Lattice C */ +X#define MCH_AMIGA +X#define MPU68000 +X#endif +X +X#ifdef unix +X#include <pwd.h> +X#include <sys/types.h> +X#include <sys/stat.h> +X#endif +X +X#define SUMSIZE 64 +X#define DEC(c) (((c) - ' ') & 077) /* single character decode */ +X +Xmain(argc, argv) +Xchar **argv; +X{ +XFILE *in, *out; +Xint through_loop=0; /* Dejavu indicator */ +Xint mode; /* file's mode (from header) */ +Xlong filesize; /* theoretical file size (from header) */ +Xchar dest[128]; +Xchar buf[80]; +X +X#ifdef AMIGA_LATTICE +Xextern int Enable_Abort; +X Enable_Abort=1; +X#endif +X +X /* A filename can be specified to be uudecoded, or nothing can +X be specified, and the input will come from STDIN */ +X +X switch (argc) +X { +X case 1: +X in=stdin; +X break; +X +X case 2: +X if ((in = fopen(argv[1], "r")) == NULL) +X { +X fprintf(stderr, "ERROR: can't find %s\n", argv[1]); +X fprintf(stderr, "USAGE: uudecode [infile]\n"); +X exit(10); +X } +X break; +X +X default: +X fprintf(stderr, "USAGE: uudecode [infile]\n"); +X exit(11); +X break; +X } +X +X /* Loop through file, searching for headers. Decode anything with a +X header, complain if there where no headers. */ +X +Xfor (;;) +X{ +X /* search file for header line */ +X for (;;) +X { +X if (fgets(buf, sizeof buf, in) == NULL) +X { +X if (!through_loop) +X { +X fprintf(stderr, "ERROR: no `begin' line!\n"); +X exit(12); +X } +X else +X { +X exit(0); +X } +X } +X if (strncmp(buf, "begin ", 6) == 0) +X break; +X } +X sscanf(buf, "begin %o %s", &mode, dest); +X +X#ifdef unix +X /* handle ~user/file format */ +X if (dest[0] == '~') +X { +X char *sl; +X struct passwd *getpwnam(); +X char *index(); +X struct passwd *user; +X char dnbuf[100]; +X +X sl = index(dest, '/'); +X if (sl == NULL) +X { +X fprintf(stderr, "Illegal ~user\n"); +X exit(13); +X } +X *sl++ = 0; +X user = getpwnam(dest+1); +X if (user == NULL) +X { +X fprintf(stderr, "No such user as %s\n", dest); +X exit(14); +X } +X strcpy(dnbuf, user->pw_dir); +X strcat(dnbuf, "/"); +X strcat(dnbuf, sl); +X strcpy(dest, dnbuf); +X } +X#endif +X +X /* create output file */ +X if ((out = fopen(dest, "w")) == NULL) +X { +X fprintf(stderr, "ERROR: can't open output file %s\n", dest); +X exit(15); +X } +X#ifdef unix +X chmod(dest, mode); +X#endif +X +X decode(in, out, dest); +X +X if (fgets(buf, sizeof buf, in) == NULL || strncmp(buf,"end",3)) +X { /* don't be overly picky about newline ^ */ +X fprintf(stderr, "ERROR: no `end' line\n"); +X exit(16); +X } +X +X if (!(fgets(buf,sizeof buf,in) == NULL || strncmp(buf,"size ",3))) +X { +X sscanf(buf, "size %ld", &filesize); +X if (ftell(out) != filesize) +X { +X fprintf(stderr, "ERROR: file should have been %ld bytes long but was +X exit(17); +X } +X } +X through_loop = 1; +X} /* forever */ +X} /* main */ +X +X/* +X * Copy from in to out, decoding as you go. +X * If a return or newline is encountered too early in a line, it is +X * assumed that means that some editor has truncated trailing spaces. +X */ +Xdecode(in, out, dest) +XFILE *in; +XFILE *out; +Xchar *dest; +X{ +Xchar buf[81]; +Xchar *bp; +Xint nosum=0; +X#ifndef unix +Xextern errno; +X#endif +Xregister int j; +Xregister int n; +Xint checksum, line; +X +X for (line = 1; ; line++) /* for each input line */ +X { +X if (fgets(buf, sizeof buf, in) == NULL) +X { +X fprintf(stderr, "ERROR: input ended unexpectedly!\n"); +X exit(18); +X } +X +X /* Pad end of lines in case some editor truncated trailing +X spaces */ +X +X for (n=0;n<79;n++) /* search for first \r, \n or \000 */ +X { +X if (buf[n]=='\176') /* If BITNET made a twiddle, */ +X buf[n]='\136'; /* we make a caret */ +X if (buf[n]=='\r'||buf[n]=='\n'||buf[n]=='\000') +X break; +X } +X for (;n<79;n++) /* when found, fill rest of line with space */ +X { +X buf[n]=' '; +X } +X buf[79]=0; /* terminate new string */ +X +X checksum = 0; +X n = DEC(buf[0]); +X if (n <= 0) +X break; /* 0 bytes on a line?? Must be the last line */ +X +X bp = &buf[1]; +X +X /* FOUR input characters go into each THREE output charcters */ +X +X while (n >= 4) +X { +X j = DEC(bp[0]) << 2 | DEC(bp[1]) >> 4; putc(j, out); checksum += j; +X j = DEC(bp[1]) << 4 | DEC(bp[2]) >> 2; putc(j, out); checksum += j; +X j = DEC(bp[2]) << 6 | DEC(bp[3]); putc(j, out); checksum += j; +X checksum = checksum % SUMSIZE; +X bp += 4; +X n -= 3; +X } +X +X j = DEC(bp[0]) << 2 | DEC(bp[1]) >> 4; +X checksum += j; +X if (n >= 1) +X putc(j, out); +X j = DEC(bp[1]) << 4 | DEC(bp[2]) >> 2; +X checksum += j; +X if (n >= 2) +X putc(j, out); +X j = DEC(bp[2]) << 6 | DEC(bp[3]); +X checksum += j; +X if (n >= 3) +X putc(j, out); +X checksum = checksum % SUMSIZE; +X bp += 4; +X n -= 3; +X +X#ifndef unix +X /* Error checking under UNIX??? You must be kidding... */ +X /* Check if an error occured while writing to that last line */ +X if (errno) +X { +X fprintf(stderr, "ERROR: error writing to %s\n",dest); +X exit(19); +X } +X#endif +X +X /* The line has been decoded; now check that sum */ +X +X nosum |= !isspace(*bp); +X if (nosum) /* Is there a checksum at all?? */ +X { +X if (checksum != DEC(*bp)) /* Does that checksum match? */ +X { +X fprintf(stderr, "ERROR: checksum mismatch decoding %s, line %d.\ +X } +X } /* sum */ +X } /* line */ +X} /* function */ +X +X#ifdef unix +X/* +X * Return the ptr in sp at which the character c appears; +X * 0 if not found +X */ +Xchar * +Xindex(sp, c) +Xregister char *sp, c; +X{ +X do +X { +X if (*sp == c) +X return(sp); +X } +X while (*sp++); +X +X return(0); +X} +X#endif unix +X + +END_OF_FILE +echo shar: NEWLINE appended to \"'uudecode.c'\" +if test 6419 -ne `wc -c <'uudecode.c'`; then + echo shar: \"'uudecode.c'\" unpacked with wrong size! +fi +# end of 'uudecode.c' +fi +echo shar: End of shell archive. +exit 0 diff --git a/MISC/MVS/README b/MISC/MVS/README new file mode 100644 index 0000000..09741ab --- /dev/null +++ b/MISC/MVS/README @@ -0,0 +1,44 @@ +These patches and supplemental programs should allow porting to MVS or MVS/XA +in an EBCDIC envrionment, using SAS C V4.00C. + +Included are: + -rw-r--r-- 1 swl26 1573 Jul 17 14:32 README + -rw-rw-r-- 1 swl26 20861 Jul 17 13:41 diffs + -rw-rw-r-- 1 swl26 5022 Jul 17 14:00 fixit.l + -rw-rw-r-- 1 swl26 97644 Jul 17 13:42 initscan.mvs.c + -rw-rw-r-- 1 swl26 4898 Jul 17 14:08 unfixit.l + +The file "diffs" contains context diffs for changes to flex 2.3. + +The file "fixit.l" contains flex sources for a program to shorten external +variable and function names to 8 characters or less. This is required for the +"dumb" compiler linker used. + +The file "unfixit.l" reverses the changes in "fixit.l", to restore long names. +This is useful when trying to build diff files as created here. + +The file "initscan.mvs.c" is an already "flexed" version of scan.l, in an +EBCDIC environment. + +To install in an MVS environment, use patch to apply the diffs to flex 2.3, +then run "fixit" on all .c, .h, .l, .y, and .skel files. Move the files +to the MVS machine, and compile each of the .c files. (You will need a +"yacc" functional equivalent under MVS to expand parse.y in that +environment.) Link together, and the resulting flex should be ready to +go. To test, run the MVSflex -is8 -Ce on the scan.l, and you should get +back a file which is identical to initscan.mvs.c. + +Enjoy. + +Steven W. Layten +Senior Engineer +Chemical Abstracts Service +PO Box 3012 +2540 Olentangy River Road +Columbus, Ohio 43210 + ++1 614 421 3600 extension 3451 + +INET: swl26%cas.BITNET@CUNYVM.CUNY.Edu +UUCP: osu-cis!chemabs!swl26 +BITNET: swl26@cas.bitnet diff --git a/MISC/MVS/diffs b/MISC/MVS/diffs new file mode 100644 index 0000000..98dfbdf --- /dev/null +++ b/MISC/MVS/diffs @@ -0,0 +1,854 @@ +diff -c ../Makefile ./Makefile +*** ../Makefile Thu Jun 28 00:44:07 1990 +--- ./Makefile Mon Jul 16 13:57:26 1990 +*************** +*** 39,49 **** + # AUXDIR, manual pages will be installed in MANDIR with extension MANEXT. + # Raw, unformatted troff source will be installed if INSTALLMAN=man, nroff + # preformatted versions will be installed if INSTALLMAN=cat. +! DESTDIR = + BINDIR = /usr/local + LIBDIR = /usr/local/lib + AUXDIR = /usr/local/lib + MANDIR = /usr/man/manl + MANEXT = l + INSTALLMAN = man + +--- 39,50 ---- + # AUXDIR, manual pages will be installed in MANDIR with extension MANEXT. + # Raw, unformatted troff source will be installed if INSTALLMAN=man, nroff + # preformatted versions will be installed if INSTALLMAN=cat. +! DESTDIR = /projects/m751stereo/code/c/swl26 + BINDIR = /usr/local + LIBDIR = /usr/local/lib + AUXDIR = /usr/local/lib + MANDIR = /usr/man/manl ++ INCLUDEDIR = . + MANEXT = l + INSTALLMAN = man + +*************** +*** 52,58 **** + + SKELETON_FILE = $(DESTDIR)$(AUXDIR)/flex.skel + SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_FILE)\" +! CFLAGS = -O + LDFLAGS = -s + + COMPRESSION = +--- 53,59 ---- + + SKELETON_FILE = $(DESTDIR)$(AUXDIR)/flex.skel + SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"$(SKELETON_FILE)\" +! CFLAGS = -O -DUSG -I$(INCLUDEDIR) + LDFLAGS = -s + + COMPRESSION = +diff -c ../ccl.c ./ccl.c +*** ../ccl.c Thu Jun 28 00:44:07 1990 +--- ./ccl.c Mon Jul 16 13:57:27 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/ccl.c,v 2.5 90/06/27 23:48:13 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + /* ccladd - add a single character to a ccl + * +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: ccl.c,v 2.5 90/06/27 23:48:13 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + /* ccladd - add a single character to a ccl + * +diff -c ../dfa.c ./dfa.c +*** ../dfa.c Thu Jun 28 00:44:08 1990 +--- ./dfa.c Mon Jul 16 13:57:28 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/dfa.c,v 2.7 90/06/27 23:48:15 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + + /* declare functions that have forward references */ +--- 28,38 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: dfa.c,v 2.7 90/06/27 23:48:15 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> +! #include <ctype.h> + + + /* declare functions that have forward references */ +*************** +*** 682,688 **** + register int j; + + for ( i = 'A', j = 'a'; i <= 'Z'; ++i, ++j ) +! state[i] = state[j]; + } + + if ( ds > num_start_states ) +--- 683,692 ---- + register int j; + + for ( i = 'A', j = 'a'; i <= 'Z'; ++i, ++j ) +! { +! if (isupper(i) ) +! state[i] = state[j]; +! } + } + + if ( ds > num_start_states ) +*************** +*** 958,964 **** + } + } + +! else if ( sym >= 'A' && sym <= 'Z' && caseins ) + flexfatal( "consistency check failed in symfollowset" ); + + else if ( sym == SYM_EPSILON ) +--- 962,968 ---- + } + } + +! else if ( isupper ( sym ) && caseins ) + flexfatal( "consistency check failed in symfollowset" ); + + else if ( sym == SYM_EPSILON ) +Only in .: diffs +Only in .: diffs.new +diff -c ../ecs.c ./ecs.c +*** ../ecs.c Thu Jun 28 00:44:08 1990 +--- ./ecs.c Mon Jul 16 13:57:28 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/ecs.c,v 2.5 90/06/27 23:48:17 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + /* ccl2ecl - convert character classes to set of equivalence classes + * +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: ecs.c,v 2.5 90/06/27 23:48:17 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + /* ccl2ecl - convert character classes to set of equivalence classes + * +diff -c ../flex.skel ./flex.skel +*** ../flex.skel Thu Jun 28 00:44:27 1990 +--- ./flex.skel Mon Jul 16 13:57:29 1990 +*************** +*** 1,7 **** + /* A lexical scanner generated by flex */ + + /* scanner skeleton version: +! * $Header: /usr/fsys/odin/a/vern/flex/RCS/flex.skel,v 2.13 90/05/26 17:24:13 vern Exp $ + */ + + #define FLEX_SCANNER +--- 1,7 ---- + /* A lexical scanner generated by flex */ + + /* scanner skeleton version: +! * $Header: flex.skel,v 2.13 90/05/26 17:24:13 vern Exp $ + */ + + #define FLEX_SCANNER +diff -c ../flexdef.h ./flexdef.h +*** ../flexdef.h Thu Jun 28 00:44:27 1990 +--- ./flexdef.h Mon Jul 16 13:57:30 1990 +*************** +*** 26,32 **** + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +! /* @(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/flexdef.h,v 2.7 90/06/27 23:48:19 vern Exp $ (LBL) */ + + #ifndef FILE + #include <stdio.h> +--- 26,32 ---- + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +! /* @(#) $Header: flexdef.h,v 2.7 90/06/27 23:48:19 vern Exp $ (LBL) */ + + #ifndef FILE + #include <stdio.h> +*************** +*** 45,51 **** + + /* size of input alphabet - should be size of ASCII set */ + #ifndef DEFAULT_CSIZE +! #define DEFAULT_CSIZE 128 + #endif + + #ifndef PROTO +--- 45,51 ---- + + /* size of input alphabet - should be size of ASCII set */ + #ifndef DEFAULT_CSIZE +! #define DEFAULT_CSIZE 256 + #endif + + #ifndef PROTO +*************** +*** 90,96 **** +--- 90,98 ---- + #define SHORT_FILE_NAMES + #endif + ++ #ifndef OSVS + char *malloc(), *realloc(); ++ #endif + + + /* maximum line length we'll have to deal with */ +*************** +*** 116,125 **** + #define true 1 + #define false 0 + +- + #ifndef DEFAULT_SKELETON_FILE + #define DEFAULT_SKELETON_FILE "flex.skel" +! #endif + + /* special chk[] values marking the slots taking by end-of-buffer and action + * numbers +--- 118,132 ---- + #define true 1 + #define false 0 + + #ifndef DEFAULT_SKELETON_FILE ++ #ifdef OSVS ++ #define DEFAULT_SKELETON_FILE "ctri01" ++ #define SYSUT1 "sysut1" ++ #define SYSUT2 "sysut2" ++ #else + #define DEFAULT_SKELETON_FILE "flex.skel" +! #endif /* OSVS */ +! #endif /* DEFAULT_SKELETON_FILE */ + + /* special chk[] values marking the slots taking by end-of-buffer and action + * numbers +*************** +*** 226,233 **** + #define INITIAL_MAX_SCS 40 /* maximum number of start conditions */ + #define MAX_SCS_INCREMENT 40 /* amount to bump by if it's not enough */ + +! #define ONE_STACK_SIZE 500 /* stack of states with only one out-transition */ +! #define SAME_TRANS -1 /* transition is the same as "default" entry for state */ + + /* the following percentages are used to tune table compression: + +--- 233,240 ---- + #define INITIAL_MAX_SCS 40 /* maximum number of start conditions */ + #define MAX_SCS_INCREMENT 40 /* amount to bump by if it's not enough */ + +! #define ONE_STACK_SIZE 500 /*stack of states with only one out-transition*/ +! #define SAME_TRANS -1 /*transition is the same as "default" entry for state */ + + /* the following percentages are used to tune table compression: + +diff -c ../gen.c ./gen.c +*** ../gen.c Thu Jun 28 00:44:28 1990 +--- ./gen.c Mon Jul 16 13:57:32 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/gen.c,v 2.9 90/06/27 23:48:22 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + + /* declare functions that have forward references */ +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: gen.c,v 2.9 90/06/27 23:48:22 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + + /* declare functions that have forward references */ +*************** +*** 292,298 **** + + indent_puts( "{" ); + +! indent_puts( "if ( yy_lp && yy_lp < yy_accept[yy_current_state + 1] )" ); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_act = yy_acclist[yy_lp];" ); +--- 292,298 ---- + + indent_puts( "{" ); + +! indent_puts("if( yy_lp && yy_lp < yy_accept[yy_current_state + 1] )"); + indent_up(); + indent_puts( "{" ); + indent_puts( "yy_act = yy_acclist[yy_lp];" ); +diff -c ../initscan.c ./initscan.c +*** ../initscan.c Thu Jun 28 00:44:51 1990 +--- ./initscan.c Mon Jul 16 13:57:33 1990 +*************** +*** 1,7 **** + /* A lexical scanner generated by flex */ + + /* scanner skeleton version: +! * $Header: /usr/fsys/odin/a/vern/flex/RCS/flex.skel,v 2.13 90/05/26 17:24:13 vern Exp $ + */ + + #define FLEX_SCANNER +--- 1,7 ---- + /* A lexical scanner generated by flex */ + + /* scanner skeleton version: +! * $Header: flex.skel,v 2.13 90/05/26 17:24:13 vern Exp $ + */ + + #define FLEX_SCANNER +*************** +*** 193,199 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/scan.l,v 2.9 90/06/27 23:48:34 vern Exp $ (LBL)"; + #endif + + #undef yywrap +--- 193,199 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: scan.l,v 2.9 90/06/27 23:48:34 vern Exp $ (LBL)"; + #endif + + #undef yywrap +diff -c ../libmain.c ./libmain.c +*** ../libmain.c Thu Jun 28 00:44:28 1990 +--- ./libmain.c Mon Jul 16 13:57:34 1990 +*************** +*** 1,6 **** + /* libmain - flex run-time support library "main" function */ + +! /* $Header: /usr/fsys/odin/a/vern/flex/RCS/libmain.c,v 1.2 90/05/26 16:50:08 vern Exp $ */ + + extern int yylex(); + +--- 1,6 ---- + /* libmain - flex run-time support library "main" function */ + +! /* $Header: libmain.c,v 1.2 90/05/26 16:50:08 vern Exp $ */ + + extern int yylex(); + +diff -c ../main.c ./main.c +*** ../main.c Thu Jun 28 00:44:29 1990 +--- ./main.c Mon Jul 16 13:57:34 1990 +*************** +*** 34,44 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/main.c,v 2.9 90/06/27 23:48:24 vern Exp $ (LBL)"; + #endif + + +! #include "flexdef.h" + + static char flex_version[] = "2.3"; + +--- 34,44 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: main.c,v 2.9 90/06/27 23:48:24 vern Exp $ (LBL)"; + #endif + + +! #include <flexdef.h> + + static char flex_version[] = "2.3"; + +*************** +*** 97,106 **** + char *program_name; + + #ifndef SHORT_FILE_NAMES + static char *outfile = "lex.yy.c"; +! #else + static char *outfile = "lexyy.c"; +! #endif + static int outfile_created = 0; + static int use_stdout; + static char *skelname = NULL; +--- 97,110 ---- + char *program_name; + + #ifndef SHORT_FILE_NAMES ++ #ifdef OSVS ++ static char *outfile = "ctro01"; ++ #else /* not OSVS */ + static char *outfile = "lex.yy.c"; +! #endif /* not OSVS */ +! #else /* SHORT_FILE_NAMES */ + static char *outfile = "lexyy.c"; +! #endif /* SHORT_FILE_NAMES */ + static int outfile_created = 0; + static int use_stdout; + static char *skelname = NULL; +*************** +*** 209,216 **** +--- 213,222 ---- + else if ( fclose( temp_action_file ) ) + flexfatal( "error occurred when closing temporary action file" ); + ++ #ifndef OSVS + else if ( unlink( action_file_name ) ) + flexfatal( "error occurred when deleting temporary action file" ); ++ #endif + } + + if ( status != 0 && outfile_created ) +*************** +*** 221,228 **** +--- 227,236 ---- + else if ( fclose( stdout ) ) + flexfatal( "error occurred when closing output file" ); + ++ #ifndef OSVS + else if ( unlink( outfile ) ) + flexfatal( "error occurred when deleting output file" ); ++ #endif + } + + if ( backtrack_report && backtrack_file ) +*************** +*** 574,583 **** + if ( backtrack_report ) + { + #ifndef SHORT_FILE_NAMES + backtrack_file = fopen( "lex.backtrack", "w" ); +! #else + backtrack_file = fopen( "lex.bck", "w" ); +! #endif + + if ( backtrack_file == NULL ) + flexerror( "could not create lex.backtrack" ); +--- 582,595 ---- + if ( backtrack_report ) + { + #ifndef SHORT_FILE_NAMES ++ #ifdef OSVS ++ backtrack_file = fopen( SYSUT2, "w"); ++ #else /* not OSVS */ + backtrack_file = fopen( "lex.backtrack", "w" ); +! #endif /* OSVS */ +! #else /* SHORT_FILE_NAMES */ + backtrack_file = fopen( "lex.bck", "w" ); +! #endif /* SHORT_FILE_NAMES */ + + if ( backtrack_file == NULL ) + flexerror( "could not create lex.backtrack" ); +*************** +*** 597,604 **** + lerrsf( "can't open skeleton file %s", skelname ); + + #ifdef SYS_V + action_file_name = tmpnam( NULL ); +! #endif + + if ( action_file_name == NULL ) + { +--- 609,620 ---- + lerrsf( "can't open skeleton file %s", skelname ); + + #ifdef SYS_V ++ #ifndef OSVS + action_file_name = tmpnam( NULL ); +! #else /* OSVS */ +! action_file_name = SYSUT1; +! #endif /* OSVS */ +! #endif /* SYS_V */ + + if ( action_file_name == NULL ) + { +*************** +*** 609,615 **** +--- 625,636 ---- + #else + (void) strcpy( temp_action_file_name, "flexXXXXXX.tmp" ); + #endif ++ #ifndef OSVS + (void) mktemp( temp_action_file_name ); ++ #else /* OSVS */ ++ /* should never be executed in OSVS as IF should always be false */ ++ (void) strcpy( temp_action_file_name, SYSUT1 ) ; ++ #endif /* OSVS */ + + action_file_name = temp_action_file_name; + } +diff -c ../misc.c ./misc.c +*** ../misc.c Thu Jun 28 00:44:40 1990 +--- ./misc.c Mon Jul 16 13:57:35 1990 +*************** +*** 28,38 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/misc.c,v 2.7 90/06/27 23:48:27 vern Exp $ (LBL)"; + #endif + + #include <ctype.h> +! #include "flexdef.h" + + + /* ANSI C does not guarantee that isascii() is defined */ +--- 28,38 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: misc.c,v 2.7 90/06/27 23:48:27 vern Exp $ (LBL)"; + #endif + + #include <ctype.h> +! #include <flexdef.h> + + + /* ANSI C does not guarantee that isascii() is defined */ +*************** +*** 107,113 **** +--- 107,117 ---- + { + while ( *str ) + { ++ #ifdef OSVS ++ if ( ! islower( *str ) ) ++ #else + if ( ! isascii( *str ) || ! islower( *str ) ) ++ #endif + return ( 0 ); + ++str; + } +*************** +*** 130,136 **** +--- 134,144 ---- + { + while ( *str ) + { ++ #ifdef OSVS ++ if ( ! isupper( (char) *str ) ) ++ #else + if ( ! isascii( *str ) || ! isupper( (char) *str ) ) ++ #endif + return ( 0 ); + ++str; + } +*************** +*** 182,188 **** +--- 190,200 ---- + register int c; + + { ++ #ifdef OSVS ++ return ( isupper( c ) ? (Char) tolower( c ) : (Char) c ); ++ #else + return ( (isascii( c ) && isupper( c )) ? tolower( c ) : c ); ++ #endif + } + + +*************** +*** 204,210 **** + for ( c = str; *c; ++c ) + ; + +! copy = malloc( (unsigned) ((c - str + 1) * sizeof( char )) ); + + if ( copy == NULL ) + flexfatal( "dynamic memory failure in copy_string()" ); +--- 216,222 ---- + for ( c = str; *c; ++c ) + ; + +! copy = (char *) malloc( (unsigned) ((c - str + 1) * sizeof( char )) ); + + if ( copy == NULL ) + flexfatal( "dynamic memory failure in copy_string()" ); +*************** +*** 392,403 **** +--- 404,421 ---- + + #ifndef MS_DOS + #ifndef VMS ++ #ifndef OSVS + #include <sys/types.h> ++ #endif /* OSVS */ + #else + #include <types.h> + #endif + #endif + ++ #ifdef OSVS ++ #include <time.h> ++ #endif /* OSVS */ ++ + #ifdef MS_DOS + #include <time.h> + typedef long time_t; +*************** +*** 615,621 **** +--- 633,643 ---- + if ( array[1] == 'x' ) + ++sptr; + ++ #ifdef OSVS ++ while ( isdigit( array[sptr] ) ) ++ #else + while ( isascii( array[sptr] ) && isdigit( array[sptr] ) ) ++ #endif + /* don't increment inside loop control because if + * isdigit() is a macro it will expand it to two + * increments ... +Only in ..: new +diff -c ../nfa.c ./nfa.c +*** ../nfa.c Thu Jun 28 00:44:40 1990 +--- ./nfa.c Mon Jul 16 13:57:36 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/nfa.c,v 2.6 90/06/27 23:48:29 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + + /* declare functions that have forward references */ +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: nfa.c,v 2.6 90/06/27 23:48:29 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + + /* declare functions that have forward references */ +*************** +*** 110,116 **** + { + int sym, tsp1, tsp2, anum, ns; + +! fprintf( stderr, "\n\n********** beginning dump of nfa with start state %d\n", + state1 ); + + /* we probably should loop starting at firstst[state1] and going to +--- 110,116 ---- + { + int sym, tsp1, tsp2, anum, ns; + +! fprintf(stderr,"\n\n********* beginning dump of nfa with start state %d\n", + state1 ); + + /* we probably should loop starting at firstst[state1] and going to +diff -c ../parse.y ./parse.y +*** ../parse.y Thu Jun 28 00:44:40 1990 +--- ./parse.y Mon Jul 16 13:57:36 1990 +*************** +*** 32,45 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/parse.y,v 2.7 90/06/27 23:48:31 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + int pat, scnum, eps, headcnt, trailcnt, anyccl, lastchar, i, actvp, rulelen; + int trlcontxt, xcluflg, cclsorted, varlength, variable_trail_rule; + Char clower(); + + static int madeany = false; /* whether we've made the '.' character class */ + int previous_continued_action; /* whether the previous rule's action was '|' */ +--- 32,47 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: parse.y,v 2.7 90/06/27 23:48:31 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + int pat, scnum, eps, headcnt, trailcnt, anyccl, lastchar, i, actvp, rulelen; + int trlcontxt, xcluflg, cclsorted, varlength, variable_trail_rule; + Char clower(); ++ void build_eof_action(); ++ void yyerror(); + + static int madeany = false; /* whether we've made the '.' character class */ + int previous_continued_action; /* whether the previous rule's action was '|' */ +diff -c ../scan.l ./scan.l +*** ../scan.l Thu Jun 28 00:44:41 1990 +--- ./scan.l Mon Jul 16 13:57:37 1990 +*************** +*** 30,42 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/scan.l,v 2.9 90/06/27 23:48:34 vern Exp $ (LBL)"; + #endif + + #undef yywrap + +! #include "flexdef.h" +! #include "parse.h" + + #define ACTION_ECHO fprintf( temp_action_file, "%s", yytext ) + #define MARK_END_OF_PROLOG fprintf( temp_action_file, "%%%% end of prolog\n" ); +--- 30,42 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: scan.l,v 2.9 90/06/27 23:48:34 vern Exp $ (LBL)"; + #endif + + #undef yywrap + +! #include <flexdef.h> +! #include <parse.h> + + #define ACTION_ECHO fprintf( temp_action_file, "%s", yytext ) + #define MARK_END_OF_PROLOG fprintf( temp_action_file, "%%%% end of prolog\n" ); +diff -c ../sym.c ./sym.c +*** ../sym.c Thu Jun 28 00:44:41 1990 +--- ./sym.c Mon Jul 16 13:57:37 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/sym.c,v 2.4 90/06/27 23:48:36 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + + /* declare functions that have forward references */ +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: sym.c,v 2.4 90/06/27 23:48:36 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + + /* declare functions that have forward references */ +diff -c ../tblcmp.c ./tblcmp.c +*** ../tblcmp.c Thu Jun 28 00:44:41 1990 +--- ./tblcmp.c Mon Jul 16 13:57:38 1990 +*************** +*** 28,37 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/tblcmp.c,v 2.5 90/06/27 23:48:38 vern Exp $ (LBL)"; + #endif + +! #include "flexdef.h" + + + /* declarations for functions that have forward references */ +--- 28,37 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: tblcmp.c,v 2.5 90/06/27 23:48:38 vern Exp $ (LBL)"; + #endif + +! #include <flexdef.h> + + + /* declarations for functions that have forward references */ +diff -c ../yylex.c ./yylex.c +*** ../yylex.c Thu Jun 28 00:44:41 1990 +--- ./yylex.c Mon Jul 16 13:57:38 1990 +*************** +*** 28,39 **** + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/yylex.c,v 2.5 90/06/27 23:48:40 vern Exp $ (LBL)"; + #endif + + #include <ctype.h> +! #include "flexdef.h" +! #include "parse.h" + + + /* ANSI C does not guarantee that isascii() is defined */ +--- 28,39 ---- + + #ifndef lint + static char rcsid[] = +! "@(#) $Header: yylex.c,v 2.5 90/06/27 23:48:40 vern Exp $ (LBL)"; + #endif + + #include <ctype.h> +! #include <flexdef.h> +! #include <parse.h> + + + /* ANSI C does not guarantee that isascii() is defined */ +*************** +*** 180,186 **** +--- 180,190 ---- + break; + + default: ++ #ifdef OSVS ++ if ( ! isprint( yylval ) ) ++ #else + if ( ! isascii( yylval ) || ! isprint( yylval ) ) ++ #endif + fprintf( stderr, "\\%.3o", yylval ); + else + (void) putc( yylval, stderr ); diff --git a/MISC/MVS/fixit.l b/MISC/MVS/fixit.l new file mode 100644 index 0000000..3e15b6f --- /dev/null +++ b/MISC/MVS/fixit.l @@ -0,0 +1,138 @@ +%{ +/* fixit.l - convert long external names to names of 8-bytes or less */ +/* + * This program is included to satisfy "dumb" compilers/linkers which + * do not know about externals of names longer than 8 bytes. + * + * Steven W. Layten + * Chemical Abstracts Service + * PO BOX 3012 + * Columbus, OH 43210 + */ +%} +%% +"action_file_name" printf("actfilnm"); +"action_out" printf("actnout"); +"add_accept" printf("addacpt"); +"all_lower" printf("alllower"); +"all_upper" printf("allupper"); +"allocate_array" printf("allocarr"); +"assoc_rule" printf("asscrule"); +"backtrack_file" printf("bktrkfil"); +"backtrack_report" printf("bktrkrep"); +"bol_needed" printf("bol_nded"); +"build_eof_action" printf("bldeofac"); +"cclinstal" printf("cclnstal"); +"ccllookup" printf("ccllookp"); +"cclnegate" printf("cclnegat"); +"cclsorted" printf("cclsrted"); +"check_for_backtracking" printf("ck4bktrk"); +"check_trailing_context" printf("cktrlcnt"); +"continued_action" printf("cntdactn"); +"copy_string" printf("copystrn"); +"copy_unsigned_string" printf("cpunsstr"); +"copyright" printf("cpyrght"); +"copysingl" printf("copysngl"); +"current_max_ccl_tbl_size" printf("c_mx_ccl"); +"current_max_dfa_size" printf("c_mx_dfa"); +"current_max_dfas" printf("c_mxdfas"); +"current_max_rules" printf("curmxrls"); +"current_max_scs" printf("c_mx_scs"); +"current_max_template_xpairs" printf("c_mx_tmp"); +"current_max_xpairs" printf("c_mx_xpr"); +"current_maxccls" printf("c_mxccls"); +"current_mns" printf("curr_mns"); +"current_state_type" printf("cursttyp"); +"dataflush" printf("datflush"); +"dfaacc_union" printf("dfacunin"); +"do_indent" printf("do_indnt"); +"dump_associated_rules" printf("dmpasrl"); +"dump_transitions" printf("dmptrns"); +"dupmachine" printf("dupmach"); +"ecs_from_xlation" printf("ecsfrmxt"); +"end_of_buffer_state" printf("eobstate"); +"epsclosure" printf("epsclos"); +"expand_nxt_chk" printf("expnxtck"); +"find_table_space" printf("fndtblsp"); +"finish_rule" printf("fnshrule"); +"firstfree" printf("firstfre"); +"firstprot" printf("firstprt"); +"flex_gettime" printf("flxgettm"); +"flexerror" printf("flxerror"); +"flexfatal" printf("flxfatal"); +"format_pinpoint_message" printf("fmtptmsg"); +"gen_NUL_trans" printf("gnNULtrn"); +"gen_backtracking" printf("gnbktrkg"); +"gen_bt_action" printf("gnbtactn"); +"gen_find_action" printf("gnfndact"); +"gen_line_dirs" printf("gnlindir"); +"gen_next_compressed_state" printf("gnnxcste"); +"gen_next_match" printf("gnnxmtch"); +"gen_next_state" printf("gnnxtst"); +"gen_start_state" printf("gnstrtst"); +"hash_entry" printf("hshentry"); +"hashfunct" printf("hshfct"); +"increase_max_dfas" printf("incmxdfa"); +"indent_put2s" printf("indput2s"); +"indent_puts" printf("indputs"); +"infilename" printf("infilnam"); +"input_files" printf("inp_fles"); +"interactive" printf("intractv"); +"line_directive_out" printf("lndirout"); +"link_machines" printf("lnkmchns"); +"list_character_set" printf("lst_cset"); +"make_tables" printf("maketbls"); +"mark_beginning_as_normal" printf("mkbgnorm"); +"mktemplate" printf("mktmplat"); +"num_backtracking" printf("nbktrckg"); +"num_input_files" printf("ninfiles"); +"num_reallocs" printf("numraloc"); +"num_rules" printf("numrules"); +"num_xlations" printf("nuxlatns"); +"numsnpairs" printf("numnpair"); +"output_file_name" printf("outfilnm"); +"peakpairs" printf("peakpair"); +"performance_report" printf("perf_rep"); +"pinpoint_message" printf("pptmsg"); +"place_state" printf("plcstate"); +"previous_continued_action" printf("prvctdan"); +"printstats" printf("prtstats"); +"program_name" printf("pgm_name"); +"protcomst" printf("prtcomst"); +"readable_form" printf("rdblefrm"); +"real_reject" printf("realrjct"); +"reallocate_array" printf("rallocar"); +"reject_really_used" printf("rjctused"); +"rule_linenum" printf("rulelnno"); +"rule_type" printf("ruletype"); +"set_input_file" printf("stinpfle"); +"set_up_initial_allocations" printf("setupia"); +"starttime" printf("startime"); +"state_type" printf("ste_type"); +"symfollowset" printf("symfollo"); +"sympartition" printf("sympartn"); +"syntaxerror" printf("syntxerr"); +"temp_action_file" printf("tmpactfl"); +"todo_head" printf("todohead"); +"todo_next" printf("todonext"); +"transchar" printf("trnschar"); +"transition_struct_out" printf("trnstout"); +"trlcontxt" printf("trlcntxt"); +"variable_trail_rule" printf("vtrailrl"); +"variable_trailing_context_rules" printf("vtrlctrl"); +"varlength" printf("varlngth"); +"yy_create_buffer" printf("yycrbffr"); +"yy_delete_buffer" printf("yydlbffr"); +"yy_init_buffer" printf("yyinbffr"); +"yy_load_buffer_state" printf("yyldbfst"); +"yy_switch_to_buffer" printf("yyswtobf"); +"yyerrflag" printf("yyerrflg"); +"yymore_really_used" printf("yymrreus"); +"yymore_used" printf("yymrused"); +"yyrestart" printf("yyrestrt"); +. ECHO; +%% +main() +{ + yylex(); +} diff --git a/MISC/MVS/initscan-mvs.c b/MISC/MVS/initscan-mvs.c new file mode 100644 index 0000000..93c8591 --- /dev/null +++ b/MISC/MVS/initscan-mvs.c @@ -0,0 +1,2672 @@ +/* A lexical scanner generated by flex */ + +/* scanner skeleton version: + * $Header: flex.skel,v 2.13 90/05/26 17:24:13 ve + */ + +#define FLEX_SCANNER + +#include <stdio.h> + +#ifdef __STDC__ + +#ifndef DONT_HAVE_STDLIB_H +#include <stdlib.h> +#else +void *malloc( unsigned ); +void free( void* ); +#endif + +#define YY_USE_PROTOS +#define YY_USE_CONST +#endif + + +/* cfront 1.2 defines "c_plusplus" instead of "__cplusplus" */ +#ifdef c_plusplus +#ifndef __cplusplus +#define __cplusplus +#endif +#endif + + +#ifdef __cplusplus + +#ifndef __STDC__ +#include <stdlib.h> +#endif + +#include <osfcn.h> + +/* use prototypes in function declarations */ +#define YY_USE_PROTOS + +/* the "const" storage-class-modifier is valid */ +#define YY_USE_CONST + +#endif + + +#ifdef __TURBOC__ +#define YY_USE_CONST +#endif + + +#ifndef YY_USE_CONST +#define const +#endif + + +#ifdef YY_USE_PROTOS +#define YY_PROTO(proto) proto +#else +#define YY_PROTO(proto) () +/* there's no standard place to get these definitions */ +char *malloc(); +int free(); +int read(); +#endif + + +/* amount of stuff to slurp up with each read */ +#ifndef YY_READ_BUF_SIZE +#define YY_READ_BUF_SIZE 8192 +#endif + +/* returned upon end-of-file */ +#define YY_END_TOK 0 + +/* copy whatever the last rule matched to the standard output */ + +/* cast to (char *) is because for 8-bit chars, yytext is (unsigned char *) */ +/* this used to be an fputs(), but since the string might contain NUL's, + * we now use fwrite() + */ +#define ECHO (void) fwrite( (char *) yytext, yyleng, 1, yyout ) + +/* gets input and stuffs it into "buf". number of characters read, or YY_NULL, + * is returned in "result". + */ +#define YY_INPUT(buf,result,max_size) \ + if ( (result = read( fileno(yyin), (char *) buf, max_size )) < 0 ) \ + YY_FATAL_ERROR( "read() in flex scanner failed" ); +#define YY_NULL 0 + +/* no semi-colon after return; correct usage is to write "yyterminate();" - + * we don't want an extra ';' after the "return" because that will cause + * some compilers to complain about unreachable statements. + */ +#define yyterminate() return ( YY_NULL ) + +/* report a fatal error */ + +/* The funky do-while is used to turn this macro definition into + * a single C statement (which needs a semi-colon terminator). + * This avoids problems with code like: + * + * if ( something_happens ) + * YY_FATAL_ERROR( "oops, the something happened" ); + * else + * everything_okay(); + * + * Prior to using the do-while the compiler would get upset at the + * "else" because it interpreted the "if" statement as being all + * done when it reached the ';' after the YY_FATAL_ERROR() call. + */ + +#define YY_FATAL_ERROR(msg) \ + do \ + { \ + (void) fputs( msg, stderr ); \ + (void) putc( '\n', stderr ); \ + exit( 1 ); \ + } \ + while ( 0 ) + +/* default yywrap function - always treat EOF as an EOF */ +#define yywrap() 1 + +/* enter a start condition. This macro really ought to take a parameter, + * but we do it the disgusting crufty way forced on us by the ()-less + * definition of BEGIN + */ +#define BEGIN yy_start = 1 + 2 * + +/* action number for EOF rule of a given start state */ +#define YY_STATE_EOF(state) (YY_END_OF_BUFFER + state + 1) + +/* special action meaning "start processing a new file" */ +#define YY_NEW_FILE \ + do \ + { \ + yyinbffr( yy_current_buffer, yyin ); \ + yyldbfst(); \ + } \ + while ( 0 ) + +/* default declaration of generated scanner - a define so the user can + * easily add parameters + */ +#define YY_DECL int yylex YY_PROTO(( void )) + +/* code executed at the end of each rule */ +#define YY_BREAK break; + +#define YY_END_OF_BUFFER_CHAR 0 + +#ifndef YY_BUF_SIZE +#define YY_BUF_SIZE (YY_READ_BUF_SIZE * 2) /* size of default input buffer */ +#endif + +typedef struct yy_buffer_state *YY_BUFFER_STATE; + +#define YY_CHAR unsigned char +# line 1 "<stdin>" +#define INITIAL 0 +/* scan.l - scanner for flex input */ +# line 5 "<stdin>" +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms with or without + * modification are permitted provided that: (1) source distributions retain + * this entire copyright notice and comment, and (2) distributions including + * binaries display the following acknowledgement: ``This product includes + * software developed by the University of California, Berkeley and its + * contributors'' in the documentation or other materials provided with the + * distribution and in all advertising materials mentioning features or use + * of this software. Neither the name of the University nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +#ifndef lint +static char rcsid[] = + "@(#) $Header: scan.l,v 2.9 90/06/27 23:48:34 vern Exp $ (LBL)"; +#endif + +#undef yywrap + +#include <flexdef.h> +#include <parse.h> + +#define ACTION_ECHO fprintf( tmpactfl, "%s", yytext ) +#define MARK_END_OF_PROLOG fprintf( tmpactfl, "%%%% end of prolog\n" ); + +#undef YY_DECL +#define YY_DECL \ + int flexscan() + +#define RETURNCHAR \ + yylval = yytext[0]; \ + return ( CHAR ); + +#define RETURNNAME \ + (void) strcpy( nmstr, (char *) yytext ); \ + return ( NAME ); + +#define PUT_BACK_STRING(str, start) \ + for ( i = strlen( (char *) (str) ) - 1; i >= start; --i ) \ + unput((str)[i]) + +#define CHECK_REJECT(str) \ + if ( allupper( str ) ) \ + reject = true; + +#define CHECK_YYMORE(str) \ + if ( alllower( str ) ) \ + yymrused = true; +#define SECT2 1 +#define SECT2PROLOG 2 +#define SECT3 3 +#define CODEBLOCK 4 +#define PICKUPDEF 5 +#define SC 6 +#define CARETISBOL 7 +#define NUM 8 +#define QUOTE 9 +#define FIRSTCCL 10 +#define CCL 11 +#define ACTION 12 +#define RECOVER 13 +#define BRACEERROR 14 +#define C_COMMENT 15 +#define ACTION_COMMENT 16 +#define ACTION_STRING 17 +#define PERCENT_BRACE_ACTION 18 +#define USED_LIST 19 +#define CODEBLOCK_2 20 +#define XLATION 21 +# line 84 "<stdin>" + +/* done after the current pattern has been matched and before the + * corresponding action - sets up yytext + */ +#define YY_DO_BEFORE_ACTION \ + yytext = yy_bp; \ + yyleng = yy_cp - yy_bp; \ + yy_hold_char = *yy_cp; \ + *yy_cp = '\0'; \ + yy_c_buf_p = yy_cp; + +#define EOB_ACT_CONTINUE_SCAN 0 +#define EOB_ACT_END_OF_FILE 1 +#define EOB_ACT_LAST_MATCH 2 + +/* return all but the first 'n' matched characters back to the input stream */ +#define yyless(n) \ + do \ + { \ + /* undo effects of setting up yytext */ \ + *yy_cp = yy_hold_char; \ + yy_c_buf_p = yy_cp = yy_bp + n; \ + YY_DO_BEFORE_ACTION; /* set up yytext again */ \ + } \ + while ( 0 ) + +#define unput(c) yyunput( c, yytext ) + + +struct yy_buffer_state + { + FILE *yy_input_file; + + YY_CHAR *yy_ch_buf; /* input buffer */ + YY_CHAR *yy_buf_pos; /* current position in input buffer */ + + /* size of input buffer in bytes, not including room for EOB characters*/ + int yy_buf_size; + + /* number of characters read into yy_ch_buf, not including EOB characters */ + int yy_n_chars; + + int yy_eof_status; /* whether we've seen an EOF on this buffer */ +#define EOF_NOT_SEEN 0 + /* "pending" happens when the EOF has been seen but there's still + * some text process + */ +#define EOF_PENDING 1 +#define EOF_DONE 2 + }; + +static YY_BUFFER_STATE yy_current_buffer; + +/* we provide macros for accessing buffer states in case in the + * future we want to put the buffer states in a more general + * "scanner state" + */ +#define YY_CURRENT_BUFFER yy_current_buffer + + +/* yy_hold_char holds the character lost when yytext is formed */ +static YY_CHAR yy_hold_char; + +static int yy_n_chars; /* number of characters read into yy_ch_buf */ + + + +#ifndef YY_USER_ACTION +#define YY_USER_ACTION +#endif + +#ifndef YY_USER_INIT +#define YY_USER_INIT +#endif + +extern YY_CHAR *yytext; +extern int yyleng; +extern FILE *yyin, *yyout; + +YY_CHAR *yytext; +int yyleng; + +FILE *yyin = (FILE *) 0, *yyout = (FILE *) 0; + +#define YY_END_OF_BUFFER 121 +typedef int yy_ste_type; +static const short int yy_accept[341] = + { 0, + 0, 0, 0, 0, 0, 0, 119, 119, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 121, 19, 7, 18, 19, 16, + 1, 17, 19, 19, 15, 19, 67, 59, 60, 66, + 51, 67, 53, 67, 67, 67, 50, 49, 52, 67, + 120, 47, 119, 119, 28, 29, 28, 28, 28, 28, + 31, 30, 32, 73, 120, 69, 72, 70, 74, 88, + 89, 86, 87, 85, 75, 77, 76, 75, 81, 81, + + 80, 81, 83, 83, 84, 83, 99, 104, 105, 100, + 105, 103, 100, 100, 97, 98, 120, 33, 91, 90, + 22, 24, 23, 107, 109, 108, 111, 113, 114, 115, + 95, 95, 96, 95, 95, 95, 95, 38, 35, 34, + 38, 38, 44, 42, 45, 44, 44, 41, 41, 41, + 41, 40, 7, 18, 0, 16, 1, 17, 3, 14, + 8, 0, 12, 4, 0, 0, 5, 0, 15, 0, + 2, 59, 60, 0, 0, 0, 56, 0, 0, 55, + 55, 54, 117, 117, 117, 50, 49, 63, 50, 0, + 47, 46, 119, 119, 28, 28, 28, 28, 28, 31, + + 30, 72, 71, 85, 78, 79, 118, 118, 118, 82, + 99, 101, 100, 0, 102, 0, 100, 100, 0, 33, + 22, 20, 107, 106, 111, 112, 95, 95, 95, 92, + 95, 95, 95, 38, 35, 38, 38, 42, 0, 43, + 43, 43, 42, 40, 0, 13, 14, 8, 8, 0, + 12, 4, 0, 0, 0, 5, 0, 6, 0, 58, + 57, 0, 64, 0, 0, 55, 55, 65, 117, 117, + 63, 28, 28, 28, 25, 0, 118, 118, 100, 100, + 0, 21, 92, 92, 95, 95, 38, 38, 0, 39, + 43, 43, 0, 11, 4, 0, 11, 0, 0, 5, + + 0, 0, 0, 117, 28, 28, 118, 100, 100, 95, + 95, 38, 38, 43, 0, 9, 0, 0, 0, 28, + 28, 100, 100, 95, 95, 38, 38, 0, 0, 26, + 27, 93, 94, 93, 94, 36, 37, 10, 62, 0 + } ; + +static const YY_CHAR yy_ec[256] = + { 0, + 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, + 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, + 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 5, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 6, 7, 6, 6, 8, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 9, 10, 6, 1, 11, 12, 13, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 14, 15, 16, 17, + 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 18, 1, 19, 1, 20, 1, 21, 22, + 23, 24, 25, 26, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 27, 28, 16, 29, 30, 31, + 28, 16, 32, 16, 16, 16, 16, 16, 16, 16, + 16, 33, 34, 35, 16, 16, 36, 37, 16, 1, + 1, 1, 38, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 39, 1, + 1, 40, 21, 22, 23, 24, 25, 26, 16, 16, + + 16, 1, 1, 1, 1, 1, 1, 41, 27, 28, + 16, 29, 30, 31, 28, 16, 32, 1, 1, 1, + 1, 1, 1, 42, 1, 33, 34, 35, 16, 16, + 36, 37, 16, 1, 1, 1, 1, 1, 1, 43, + 43, 43, 43, 43, 43, 43, 43, 43, 43, 1, + 1, 1, 1, 1, 1 + } ; + +static const short int yy_base[404] = + { 0, + 0, 43, 85, 126, 1573, 1572, 1571, 1570, 168, 1558, + 97, 104, 211, 0, 1544, 1543, 99, 106, 118, 251, + 252, 254, 112, 114, 296, 0, 1549, 1548, 107, 111, + 140, 151, 153, 155, 253, 336, 378, 0, 339, 420, + 0, 0, 424, 465, 1550, 3243, 257, 3243, 1515, 0, + 265, 3243, 1538, 494, 0, 1540, 3243, 272, 3243, 3243, + 1495, 277, 3243, 1459, 534, 61, 343, 3243, 3243, 81, + 1496, 0, 1495, 3243, 0, 3243, 0, 1473, 1443, 1438, + 0, 281, 3243, 3243, 3243, 3243, 0, 1467, 3243, 3243, + 3243, 3243, 3243, 1434, 3243, 3243, 3243, 76, 3243, 1463, + + 3243, 242, 3243, 0, 3243, 314, 0, 3243, 1464, 0, + 330, 3243, 1448, 745, 3243, 3243, 777, 3243, 3243, 3243, + 0, 3243, 767, 0, 3243, 766, 0, 3243, 3243, 0, + 0, 350, 3243, 737, 0, 752, 739, 0, 286, 3243, + 750, 737, 3243, 357, 3243, 739, 318, 3243, 428, 738, + 327, 728, 364, 3243, 432, 0, 438, 3243, 3243, 372, + 442, 765, 446, 0, 451, 84, 0, 765, 0, 764, + 3243, 457, 3243, 763, 718, 732, 3243, 434, 438, 0, + 566, 3243, 3243, 0, 712, 480, 3243, 0, 3243, 750, + 0, 3243, 749, 3243, 0, 0, 725, 722, 609, 0, + + 484, 0, 3243, 707, 3243, 3243, 3243, 0, 706, 3243, + 0, 3243, 0, 456, 3243, 0, 721, 718, 742, 3243, + 0, 741, 0, 3243, 0, 3243, 0, 488, 703, 652, + 0, 709, 706, 0, 494, 707, 704, 499, 508, 3243, + 0, 689, 694, 688, 581, 3243, 518, 0, 603, 725, + 698, 0, 702, 693, 697, 0, 706, 3243, 705, 3243, + 3243, 671, 3243, 717, 669, 0, 0, 3243, 0, 655, + 0, 631, 573, 0, 3243, 577, 0, 537, 554, 507, + 529, 3243, 0, 0, 507, 500, 493, 485, 711, 3243, + 0, 471, 502, 3243, 0, 715, 3243, 472, 476, 0, + + 468, 740, 682, 3243, 469, 447, 3243, 455, 432, 440, + 426, 427, 414, 3243, 413, 3243, 415, 684, 688, 339, + 339, 258, 265, 238, 142, 128, 133, 121, 126, 0, + 0, 0, 0, 0, 0, 0, 0, 3243, 3243, 3243, + 783, 826, 869, 912, 955, 998, 1041, 1084, 1127, 1170, + 1213, 1256, 1299, 1342, 1385, 1428, 1460, 1503, 1535, 1578, + 1621, 1664, 1707, 1750, 1793, 1836, 1868, 1911, 1943, 1986, + 2029, 2072, 2115, 2147, 2190, 2233, 2276, 2319, 2362, 2405, + 2448, 2480, 2523, 2566, 2609, 2637, 2659, 2696, 2739, 2782, + 2805, 2848, 2871, 2914, 2937, 2980, 3012, 3044, 3067, 3110, + + 3133, 3176, 3199 + } ; + +static const short int yy_def[404] = + { 0, + 340, 340, 341, 341, 342, 342, 343, 343, 340, 9, + 344, 344, 340, 13, 345, 345, 346, 346, 347, 347, + 348, 348, 349, 349, 340, 25, 350, 350, 345, 345, + 351, 351, 352, 352, 353, 353, 340, 37, 354, 354, + 37, 37, 355, 356, 340, 340, 340, 340, 340, 357, + 340, 340, 340, 358, 359, 360, 340, 340, 340, 340, + 340, 340, 340, 361, 340, 362, 340, 340, 340, 340, + 363, 364, 365, 340, 366, 340, 367, 367, 367, 366, + 368, 340, 340, 340, 340, 340, 369, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 362, 340, 370, + + 340, 371, 340, 372, 340, 362, 373, 340, 340, 374, + 375, 340, 374, 374, 340, 340, 376, 340, 340, 340, + 377, 340, 340, 378, 340, 340, 379, 340, 340, 380, + 381, 381, 340, 381, 382, 382, 382, 383, 340, 340, + 383, 383, 340, 340, 340, 340, 384, 340, 340, 340, + 384, 340, 340, 340, 340, 357, 340, 340, 340, 340, + 385, 340, 340, 386, 340, 340, 387, 388, 359, 360, + 340, 340, 340, 389, 340, 340, 340, 361, 361, 390, + 390, 340, 340, 391, 340, 340, 340, 392, 340, 363, + 364, 340, 365, 340, 366, 367, 367, 367, 340, 368, + + 340, 369, 340, 340, 340, 340, 340, 393, 340, 340, + 373, 340, 374, 375, 340, 375, 374, 374, 376, 340, + 377, 394, 378, 340, 379, 340, 381, 381, 381, 340, + 382, 382, 382, 383, 340, 383, 383, 340, 340, 340, + 395, 340, 340, 340, 340, 340, 340, 385, 385, 396, + 340, 397, 396, 340, 340, 398, 388, 340, 389, 340, + 340, 340, 340, 361, 361, 390, 181, 340, 399, 340, + 392, 367, 367, 199, 340, 400, 401, 340, 374, 374, + 394, 340, 230, 402, 382, 382, 383, 383, 340, 340, + 403, 340, 396, 340, 397, 396, 340, 340, 340, 398, + + 340, 264, 361, 340, 367, 367, 340, 374, 374, 382, + 382, 383, 383, 340, 340, 340, 340, 361, 361, 367, + 367, 374, 374, 382, 382, 383, 383, 340, 340, 367, + 367, 374, 374, 382, 382, 383, 383, 340, 340, 0, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + + 340, 340, 340 + } ; + +static const short int yy_nxt[3287] = + { 0, + 46, 47, 47, 48, 47, 46, 46, 46, 46, 46, + 46, 46, 46, 46, 49, 50, 46, 46, 46, 46, + 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, + 50, 50, 50, 50, 50, 50, 50, 46, 46, 46, + 46, 46, 46, 46, 51, 51, 52, 51, 46, 46, + 46, 46, 46, 46, 46, 53, 46, 54, 55, 46, + 56, 46, 46, 55, 55, 55, 55, 55, 55, 55, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, + 46, 46, 46, 46, 46, 46, 58, 58, 59, 58, + 60, 61, 60, 62, 60, 188, 184, 60, 82, 82, + + 83, 82, 91, 185, 63, 82, 82, 83, 82, 91, + 119, 184, 92, 254, 119, 85, 255, 85, 185, 92, + 189, 96, 64, 104, 65, 104, 66, 67, 67, 68, + 67, 60, 61, 60, 62, 60, 69, 97, 60, 93, + 70, 94, 339, 122, 338, 63, 93, 120, 94, 123, + 105, 120, 105, 106, 122, 106, 125, 337, 125, 98, + 123, 336, 126, 64, 126, 65, 335, 66, 75, 75, + 75, 76, 75, 75, 75, 75, 75, 75, 75, 75, + 75, 75, 75, 77, 75, 75, 75, 75, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 78, + + 77, 77, 77, 77, 79, 75, 75, 75, 75, 75, + 75, 84, 84, 84, 85, 84, 84, 84, 84, 84, + 84, 84, 84, 84, 86, 84, 87, 88, 84, 84, + 84, 87, 87, 87, 87, 87, 87, 87, 87, 87, + 87, 87, 87, 87, 87, 87, 87, 87, 84, 84, + 84, 84, 84, 84, 96, 85, 128, 85, 153, 153, + 154, 153, 100, 101, 100, 101, 157, 157, 158, 157, + 97, 334, 129, 172, 172, 173, 172, 208, 177, 174, + 177, 177, 201, 201, 209, 201, 175, 235, 235, 333, + 235, 332, 98, 102, 130, 102, 107, 107, 107, 108, + + 107, 107, 107, 107, 107, 107, 107, 107, 109, 107, + 107, 110, 107, 107, 111, 112, 110, 110, 110, 110, + 110, 110, 110, 110, 110, 110, 110, 113, 110, 110, + 110, 110, 114, 107, 107, 115, 116, 107, 107, 128, + 139, 139, 140, 139, 186, 186, 187, 186, 215, 184, + 174, 228, 228, 241, 228, 129, 185, 175, 238, 238, + 242, 238, 241, 331, 229, 153, 153, 154, 153, 242, + 141, 216, 330, 247, 247, 142, 247, 130, 131, 132, + 132, 133, 132, 131, 131, 131, 131, 131, 131, 131, + 131, 131, 134, 135, 131, 131, 131, 131, 135, 135, + + 135, 135, 135, 135, 135, 135, 135, 135, 135, 136, + 135, 135, 135, 135, 137, 131, 131, 131, 131, 131, + 131, 139, 139, 140, 139, 144, 144, 145, 144, 243, + 243, 329, 243, 245, 245, 246, 245, 328, 146, 157, + 157, 158, 157, 249, 249, 327, 249, 251, 251, 326, + 251, 141, 245, 245, 246, 253, 142, 325, 172, 172, + 173, 172, 324, 323, 174, 147, 149, 149, 145, 149, + 244, 175, 263, 264, 215, 179, 178, 322, 321, 150, + 265, 186, 186, 187, 186, 201, 201, 174, 201, 228, + 228, 320, 228, 317, 175, 235, 235, 216, 235, 316, + + 238, 238, 229, 238, 315, 294, 151, 152, 161, 289, + 289, 290, 289, 240, 162, 313, 163, 312, 162, 247, + 247, 162, 247, 162, 162, 163, 164, 165, 166, 167, + 311, 310, 282, 168, 180, 180, 180, 309, 180, 180, + 180, 180, 180, 180, 180, 180, 180, 180, 180, 181, + 180, 180, 180, 180, 181, 181, 181, 181, 181, 181, + 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, + 181, 180, 180, 180, 180, 180, 182, 267, 308, 207, + 275, 267, 245, 245, 246, 245, 267, 267, 267, 267, + 267, 267, 267, 267, 267, 267, 267, 267, 267, 267, + + 267, 267, 267, 306, 249, 249, 268, 249, 267, 274, + 274, 274, 275, 274, 274, 274, 274, 274, 274, 274, + 274, 274, 274, 274, 276, 274, 274, 274, 274, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 274, 274, 274, 274, + 274, 274, 283, 283, 283, 305, 283, 283, 283, 283, + 283, 283, 283, 283, 283, 283, 283, 284, 283, 283, + 283, 283, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 283, + 283, 283, 283, 283, 283, 243, 243, 183, 243, 251, + + 251, 301, 251, 296, 296, 297, 296, 263, 260, 258, + 179, 303, 289, 289, 290, 289, 296, 296, 297, 296, + 263, 299, 263, 179, 319, 179, 263, 298, 294, 179, + 244, 292, 288, 287, 286, 285, 244, 302, 302, 302, + 302, 302, 302, 230, 282, 220, 280, 279, 278, 204, + 273, 272, 194, 191, 270, 263, 262, 261, 179, 302, + 318, 318, 318, 318, 318, 318, 260, 171, 258, 250, + 244, 239, 239, 237, 236, 233, 232, 230, 224, 222, + 220, 218, 318, 57, 57, 57, 57, 57, 57, 57, + 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, + + 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, + 57, 57, 57, 57, 57, 57, 57, 57, 57, 57, + 57, 57, 57, 57, 57, 57, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, + 71, 71, 71, 71, 71, 71, 71, 71, 71, 73, + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, + + 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, + 73, 73, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 81, 81, 81, 81, 81, + 81, 81, 81, 81, 81, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 85, 85, + 85, 85, 85, 85, 85, 85, 85, 85, 90, 90, + + 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, + 90, 90, 90, 90, 90, 90, 90, 90, 90, 90, + 90, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 95, 95, 95, 95, 95, 95, + 95, 95, 95, 95, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, + 99, 99, 99, 99, 99, 99, 99, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 103, 103, 103, 103, 103, 103, 103, 103, 103, 103, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + + 117, 117, 117, 117, 117, 117, 117, 117, 117, 117, + 117, 117, 117, 121, 121, 121, 121, 121, 121, 121, + 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, + 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, + 121, 121, 121, 121, 121, 121, 121, 121, 121, 121, + 121, 121, 121, 121, 121, 121, 124, 124, 124, 124, + 124, 124, 124, 124, 124, 124, 124, 124, 124, 124, + 124, 124, 124, 124, 124, 124, 124, 124, 124, 124, + 124, 124, 124, 124, 124, 124, 124, 124, 124, 124, + 124, 124, 124, 124, 124, 124, 124, 124, 124, 127, + + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, + 127, 127, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 138, 138, 138, 138, 138, + 138, 138, 138, 138, 138, 143, 143, 143, 143, 143, + 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, + + 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, + 143, 143, 143, 143, 143, 143, 143, 143, 143, 143, + 143, 143, 143, 143, 143, 143, 143, 143, 148, 148, + 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, + 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, + 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, + 148, 148, 148, 148, 148, 148, 148, 148, 148, 148, + 148, 156, 217, 212, 206, 156, 204, 203, 199, 198, + 156, 156, 156, 156, 156, 156, 156, 156, 156, 156, + 156, 156, 156, 156, 156, 156, 156, 197, 194, 191, + + 179, 176, 156, 160, 160, 160, 160, 160, 160, 160, + 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, + 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, + 160, 160, 160, 160, 160, 160, 160, 160, 160, 160, + 160, 160, 160, 171, 160, 160, 169, 159, 155, 340, + 169, 118, 118, 89, 89, 169, 169, 169, 169, 169, + 169, 169, 169, 169, 169, 169, 169, 169, 169, 169, + 169, 169, 80, 74, 74, 72, 72, 169, 170, 170, + 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, + 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, + + 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, + 170, 170, 170, 170, 170, 170, 170, 170, 170, 170, + 170, 178, 178, 178, 340, 178, 178, 178, 178, 178, + 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, + 178, 178, 178, 178, 178, 178, 178, 178, 178, 178, + 178, 178, 178, 178, 178, 178, 178, 178, 178, 340, + 178, 178, 178, 178, 183, 183, 183, 340, 183, 183, + 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, + 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, + 183, 183, 183, 183, 183, 183, 183, 183, 183, 183, + + 183, 183, 183, 183, 183, 183, 183, 190, 190, 190, + 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, + 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, + 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, + 190, 190, 190, 190, 190, 190, 190, 190, 190, 190, + 192, 340, 340, 340, 340, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 192, 192, 192, 192, 192, 192, 192, + 192, 192, 192, 193, 193, 193, 193, 193, 193, 193, + + 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, + 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, + 193, 193, 193, 193, 193, 193, 193, 193, 193, 193, + 193, 193, 193, 193, 193, 193, 195, 195, 195, 340, + 195, 195, 195, 195, 195, 195, 195, 195, 195, 195, + 195, 340, 195, 195, 195, 195, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 195, 195, 195, 195, 195, 195, 196, + 340, 340, 340, 196, 340, 340, 340, 340, 196, 196, + 196, 196, 196, 196, 196, 196, 196, 196, 196, 196, + + 196, 196, 196, 196, 196, 340, 340, 340, 340, 340, + 196, 200, 200, 200, 340, 200, 200, 200, 200, 200, + 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, + 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, + 200, 200, 200, 200, 200, 200, 200, 200, 200, 200, + 200, 200, 200, 200, 202, 340, 340, 340, 202, 340, + 340, 340, 340, 202, 202, 202, 202, 202, 202, 202, + 202, 202, 202, 202, 202, 202, 202, 202, 202, 202, + 340, 340, 340, 340, 340, 202, 205, 205, 205, 340, + 205, 205, 205, 205, 205, 205, 205, 205, 205, 205, + + 205, 205, 205, 205, 205, 205, 205, 205, 205, 205, + 205, 205, 205, 205, 205, 205, 205, 205, 205, 205, + 205, 205, 205, 205, 205, 205, 205, 205, 205, 207, + 207, 207, 340, 207, 207, 207, 207, 207, 207, 207, + 207, 207, 207, 207, 207, 207, 207, 207, 207, 207, + 207, 207, 207, 207, 207, 207, 207, 207, 207, 207, + 207, 207, 207, 207, 207, 207, 207, 207, 207, 207, + 207, 207, 210, 210, 210, 340, 210, 210, 210, 210, + 210, 210, 210, 210, 210, 210, 210, 210, 210, 210, + 210, 210, 210, 210, 210, 210, 210, 210, 210, 210, + + 210, 210, 210, 210, 210, 210, 210, 210, 210, 210, + 340, 210, 210, 210, 210, 211, 211, 211, 340, 211, + 211, 211, 211, 211, 211, 211, 211, 340, 211, 211, + 340, 211, 211, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 211, 211, 340, 340, 211, 211, 213, 340, + 340, 340, 213, 340, 340, 340, 340, 213, 213, 213, + 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, + 213, 213, 213, 213, 340, 340, 340, 340, 340, 213, + 214, 214, 214, 340, 214, 214, 214, 214, 214, 214, + + 214, 214, 214, 214, 214, 214, 214, 214, 214, 214, + 214, 214, 214, 214, 214, 214, 214, 214, 214, 214, + 214, 214, 214, 214, 214, 214, 214, 214, 214, 214, + 214, 214, 214, 219, 219, 219, 219, 219, 219, 219, + 219, 219, 219, 219, 219, 219, 219, 219, 219, 219, + 219, 219, 219, 219, 219, 219, 219, 219, 219, 219, + 219, 219, 219, 219, 219, 219, 219, 219, 219, 219, + 219, 219, 219, 219, 219, 219, 221, 221, 221, 340, + 221, 221, 221, 221, 221, 340, 221, 221, 221, 221, + 221, 221, 221, 221, 221, 221, 221, 221, 221, 221, + + 221, 221, 221, 221, 221, 221, 221, 221, 221, 221, + 221, 221, 221, 221, 221, 221, 221, 221, 221, 223, + 223, 223, 340, 223, 223, 223, 223, 223, 340, 223, + 223, 223, 223, 223, 223, 223, 223, 223, 223, 223, + 223, 223, 223, 223, 223, 223, 223, 223, 223, 223, + 223, 223, 223, 223, 223, 223, 223, 223, 223, 223, + 223, 223, 225, 225, 225, 340, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 225, 225, 225, 225, 225, + 225, 340, 225, 225, 225, 225, 225, 225, 225, 225, + 225, 225, 225, 225, 225, 225, 225, 225, 225, 225, + + 225, 225, 225, 340, 225, 226, 226, 226, 340, 226, + 226, 226, 226, 226, 226, 226, 226, 226, 226, 226, + 226, 226, 226, 226, 226, 226, 226, 226, 226, 226, + 226, 226, 226, 226, 226, 226, 226, 226, 226, 226, + 226, 226, 226, 226, 226, 226, 226, 226, 227, 227, + 227, 340, 227, 227, 227, 227, 227, 227, 227, 227, + 227, 227, 227, 340, 227, 227, 227, 227, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 227, 227, 227, 227, 227, + 227, 231, 340, 340, 340, 231, 340, 340, 340, 340, + + 231, 231, 231, 231, 231, 231, 231, 231, 231, 231, + 231, 231, 231, 231, 231, 231, 231, 340, 340, 340, + 340, 340, 231, 234, 340, 340, 340, 340, 234, 234, + 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, + 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, + 234, 234, 234, 234, 234, 234, 234, 234, 234, 234, + 234, 234, 234, 234, 234, 234, 240, 240, 240, 340, + 240, 240, 240, 240, 240, 240, 240, 240, 240, 240, + 240, 240, 240, 240, 240, 240, 240, 240, 240, 240, + 240, 240, 240, 240, 240, 240, 240, 240, 240, 240, + + 240, 240, 240, 240, 240, 240, 240, 240, 240, 248, + 248, 248, 340, 248, 248, 248, 248, 248, 248, 248, + 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, + 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, + 248, 248, 248, 248, 248, 248, 248, 248, 248, 248, + 248, 248, 252, 340, 340, 340, 340, 252, 252, 252, + 252, 252, 252, 252, 252, 252, 252, 252, 252, 252, + 252, 252, 252, 252, 256, 340, 340, 340, 340, 256, + 256, 256, 256, 256, 256, 256, 256, 256, 256, 256, + 256, 256, 256, 256, 256, 256, 257, 257, 257, 257, + + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 257, + 257, 257, 257, 257, 257, 257, 257, 257, 257, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 259, 259, 259, 259, 259, 259, 259, 259, + 259, 259, 266, 266, 266, 340, 266, 266, 266, 266, + 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + + 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, + 266, 266, 340, 266, 266, 269, 269, 269, 269, 269, + 269, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 269, 271, 271, + 271, 340, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 271, 271, 271, 271, 271, 271, 271, 271, 271, + 271, 277, 277, 277, 277, 277, 277, 340, 340, 340, + + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 277, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 281, 281, 281, + 281, 281, 281, 281, 281, 281, 281, 291, 291, 291, + 291, 291, 291, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 291, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 293, 293, 293, 293, 293, 293, 293, + 293, 293, 293, 295, 340, 340, 340, 295, 340, 340, + 340, 340, 295, 295, 295, 295, 295, 295, 295, 295, + 295, 295, 295, 295, 295, 295, 295, 295, 295, 340, + 340, 340, 340, 340, 295, 300, 340, 340, 340, 300, + 340, 340, 340, 340, 300, 300, 300, 300, 300, 300, + 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, + 300, 340, 340, 340, 340, 340, 300, 304, 304, 304, + 304, 304, 304, 340, 340, 340, 340, 340, 340, 340, + + 340, 340, 340, 340, 340, 340, 340, 340, 340, 304, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 276, 276, 276, 276, 276, 276, 276, + 276, 276, 276, 307, 307, 307, 307, 307, 307, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 307, 284, 284, 284, 340, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, + + 284, 284, 284, 284, 284, 284, 284, 284, 284, 284, + 284, 284, 284, 284, 284, 284, 284, 284, 284, 314, + 314, 314, 314, 314, 314, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 314, 45, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340 + } ; + +static const short int yy_chk[3287] = + { 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 70, 66, 3, 11, 11, + + 11, 11, 17, 66, 3, 12, 12, 12, 12, 18, + 29, 98, 17, 166, 30, 23, 166, 24, 98, 18, + 70, 19, 3, 23, 3, 24, 3, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 19, 4, 17, + 4, 17, 329, 31, 328, 4, 18, 29, 18, 31, + 23, 30, 24, 23, 32, 24, 33, 327, 34, 19, + 32, 326, 33, 4, 34, 4, 325, 4, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 20, 21, 35, 22, 47, 47, + 47, 47, 21, 21, 22, 22, 51, 51, 51, 51, + 20, 324, 35, 58, 58, 58, 58, 102, 62, 58, + 62, 62, 82, 82, 102, 82, 58, 139, 139, 323, + 139, 322, 20, 21, 35, 22, 25, 25, 25, 25, + + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 36, + 39, 39, 39, 39, 67, 67, 67, 67, 111, 106, + 67, 132, 132, 147, 132, 36, 106, 67, 144, 144, + 147, 144, 151, 321, 132, 153, 153, 153, 153, 151, + 39, 111, 320, 160, 160, 39, 160, 36, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 40, 40, 40, 40, 43, 43, 43, 43, 149, + 149, 317, 149, 155, 155, 155, 155, 315, 43, 157, + 157, 157, 157, 161, 161, 313, 161, 163, 163, 312, + 163, 40, 165, 165, 165, 165, 40, 311, 172, 172, + 172, 172, 310, 309, 172, 43, 44, 44, 44, 44, + 149, 172, 178, 179, 214, 178, 179, 308, 306, 44, + 179, 186, 186, 186, 186, 201, 201, 186, 201, 228, + 228, 305, 228, 301, 186, 235, 235, 214, 235, 299, + + 238, 238, 228, 238, 298, 293, 44, 44, 54, 239, + 239, 239, 239, 292, 54, 288, 54, 287, 54, 247, + 247, 54, 247, 54, 54, 54, 54, 54, 54, 54, + 286, 285, 281, 54, 65, 65, 65, 280, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, + 65, 65, 65, 65, 65, 65, 65, 181, 279, 278, + 276, 181, 245, 245, 245, 245, 181, 181, 181, 181, + 181, 181, 181, 181, 181, 181, 181, 181, 181, 181, + + 181, 181, 181, 273, 249, 249, 181, 249, 181, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 199, 199, 199, 199, 199, 199, 199, 199, + 199, 199, 230, 230, 230, 272, 230, 230, 230, 230, + 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, + 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, + 230, 230, 230, 230, 230, 230, 230, 230, 230, 230, + 230, 230, 230, 230, 230, 243, 243, 270, 243, 251, + + 251, 262, 251, 253, 253, 253, 253, 265, 259, 257, + 265, 265, 289, 289, 289, 289, 296, 296, 296, 296, + 303, 255, 318, 303, 303, 318, 319, 254, 250, 319, + 244, 242, 237, 236, 233, 232, 243, 264, 264, 264, + 264, 264, 264, 229, 222, 219, 218, 217, 209, 204, + 198, 197, 193, 190, 185, 264, 176, 175, 264, 264, + 302, 302, 302, 302, 302, 302, 174, 170, 168, 162, + 152, 150, 146, 142, 141, 137, 136, 134, 126, 123, + 117, 114, 302, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 342, 342, 342, 342, + 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, + 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, + 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, + 342, 342, 342, 342, 342, 342, 342, 342, 342, 343, + 343, 343, 343, 343, 343, 343, 343, 343, 343, 343, + 343, 343, 343, 343, 343, 343, 343, 343, 343, 343, + 343, 343, 343, 343, 343, 343, 343, 343, 343, 343, + + 343, 343, 343, 343, 343, 343, 343, 343, 343, 343, + 343, 343, 344, 344, 344, 344, 344, 344, 344, 344, + 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, + 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, + 344, 344, 344, 344, 344, 344, 344, 344, 344, 344, + 344, 344, 344, 344, 344, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, + 345, 345, 345, 345, 345, 345, 345, 345, 346, 346, + + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, + 346, 347, 347, 347, 347, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 348, 348, 348, 348, 348, 348, + 348, 348, 348, 348, 348, 348, 348, 348, 348, 348, + + 348, 348, 348, 348, 348, 348, 348, 348, 348, 348, + 348, 348, 348, 348, 348, 348, 348, 348, 348, 348, + 348, 348, 348, 348, 348, 348, 348, 349, 349, 349, + 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, + 350, 350, 350, 350, 350, 350, 350, 350, 350, 350, + 350, 350, 350, 350, 350, 350, 350, 350, 350, 350, + 350, 350, 350, 350, 350, 350, 350, 350, 350, 350, + + 350, 350, 350, 350, 350, 350, 350, 350, 350, 350, + 350, 350, 350, 351, 351, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 351, 351, 351, 351, + 351, 351, 351, 351, 351, 351, 352, 352, 352, 352, + 352, 352, 352, 352, 352, 352, 352, 352, 352, 352, + 352, 352, 352, 352, 352, 352, 352, 352, 352, 352, + 352, 352, 352, 352, 352, 352, 352, 352, 352, 352, + 352, 352, 352, 352, 352, 352, 352, 352, 352, 353, + + 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, + 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, + 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, + 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, + 353, 353, 354, 354, 354, 354, 354, 354, 354, 354, + 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, + 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, + 354, 354, 354, 354, 354, 354, 354, 354, 354, 354, + 354, 354, 354, 354, 354, 355, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, + + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 356, 356, + 356, 356, 356, 356, 356, 356, 356, 356, 356, 356, + 356, 356, 356, 356, 356, 356, 356, 356, 356, 356, + 356, 356, 356, 356, 356, 356, 356, 356, 356, 356, + 356, 356, 356, 356, 356, 356, 356, 356, 356, 356, + 356, 357, 113, 109, 100, 357, 94, 88, 80, 79, + 357, 357, 357, 357, 357, 357, 357, 357, 357, 357, + 357, 357, 357, 357, 357, 357, 357, 78, 73, 71, + + 64, 61, 357, 358, 358, 358, 358, 358, 358, 358, + 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + 358, 358, 358, 358, 358, 358, 358, 358, 358, 358, + 358, 358, 358, 56, 358, 358, 359, 53, 49, 45, + 359, 28, 27, 16, 15, 359, 359, 359, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 10, 8, 7, 6, 5, 359, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + 360, 361, 361, 361, 0, 361, 361, 361, 361, 361, + 361, 361, 361, 361, 361, 361, 361, 361, 361, 361, + 361, 361, 361, 361, 361, 361, 361, 361, 361, 361, + 361, 361, 361, 361, 361, 361, 361, 361, 361, 0, + 361, 361, 361, 361, 362, 362, 362, 0, 362, 362, + 362, 362, 362, 362, 362, 362, 362, 362, 362, 362, + 362, 362, 362, 362, 362, 362, 362, 362, 362, 362, + 362, 362, 362, 362, 362, 362, 362, 362, 362, 362, + + 362, 362, 362, 362, 362, 362, 362, 363, 363, 363, + 363, 363, 363, 363, 363, 363, 363, 363, 363, 363, + 363, 363, 363, 363, 363, 363, 363, 363, 363, 363, + 363, 363, 363, 363, 363, 363, 363, 363, 363, 363, + 363, 363, 363, 363, 363, 363, 363, 363, 363, 363, + 364, 0, 0, 0, 0, 364, 364, 364, 364, 364, + 364, 364, 364, 364, 364, 364, 364, 364, 364, 364, + 364, 364, 364, 364, 364, 364, 364, 364, 364, 364, + 364, 364, 364, 364, 364, 364, 364, 364, 364, 364, + 364, 364, 364, 365, 365, 365, 365, 365, 365, 365, + + 365, 365, 365, 365, 365, 365, 365, 365, 365, 365, + 365, 365, 365, 365, 365, 365, 365, 365, 365, 365, + 365, 365, 365, 365, 365, 365, 365, 365, 365, 365, + 365, 365, 365, 365, 365, 365, 366, 366, 366, 0, + 366, 366, 366, 366, 366, 366, 366, 366, 366, 366, + 366, 0, 366, 366, 366, 366, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 366, 366, 366, 366, 366, 366, 367, + 0, 0, 0, 367, 0, 0, 0, 0, 367, 367, + 367, 367, 367, 367, 367, 367, 367, 367, 367, 367, + + 367, 367, 367, 367, 367, 0, 0, 0, 0, 0, + 367, 368, 368, 368, 0, 368, 368, 368, 368, 368, + 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, + 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, + 368, 368, 368, 368, 368, 368, 368, 368, 368, 368, + 368, 368, 368, 368, 369, 0, 0, 0, 369, 0, + 0, 0, 0, 369, 369, 369, 369, 369, 369, 369, + 369, 369, 369, 369, 369, 369, 369, 369, 369, 369, + 0, 0, 0, 0, 0, 369, 370, 370, 370, 0, + 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, + + 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, + 370, 370, 370, 370, 370, 370, 370, 370, 370, 370, + 370, 370, 370, 370, 370, 370, 370, 370, 370, 371, + 371, 371, 0, 371, 371, 371, 371, 371, 371, 371, + 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, + 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, + 371, 371, 371, 371, 371, 371, 371, 371, 371, 371, + 371, 371, 372, 372, 372, 0, 372, 372, 372, 372, + 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + + 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + 0, 372, 372, 372, 372, 373, 373, 373, 0, 373, + 373, 373, 373, 373, 373, 373, 373, 0, 373, 373, + 0, 373, 373, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 373, 373, 0, 0, 373, 373, 374, 0, + 0, 0, 374, 0, 0, 0, 0, 374, 374, 374, + 374, 374, 374, 374, 374, 374, 374, 374, 374, 374, + 374, 374, 374, 374, 0, 0, 0, 0, 0, 374, + 375, 375, 375, 0, 375, 375, 375, 375, 375, 375, + + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 375, 375, 375, 375, 375, 375, 375, + 375, 375, 375, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 376, 376, 376, 376, + 376, 376, 376, 376, 376, 376, 377, 377, 377, 0, + 377, 377, 377, 377, 377, 0, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, + + 377, 377, 377, 377, 377, 377, 377, 377, 377, 377, + 377, 377, 377, 377, 377, 377, 377, 377, 377, 378, + 378, 378, 0, 378, 378, 378, 378, 378, 0, 378, + 378, 378, 378, 378, 378, 378, 378, 378, 378, 378, + 378, 378, 378, 378, 378, 378, 378, 378, 378, 378, + 378, 378, 378, 378, 378, 378, 378, 378, 378, 378, + 378, 378, 379, 379, 379, 0, 379, 379, 379, 379, + 379, 379, 379, 379, 379, 379, 379, 379, 379, 379, + 379, 0, 379, 379, 379, 379, 379, 379, 379, 379, + 379, 379, 379, 379, 379, 379, 379, 379, 379, 379, + + 379, 379, 379, 0, 379, 380, 380, 380, 0, 380, + 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + 380, 380, 380, 380, 380, 380, 380, 380, 381, 381, + 381, 0, 381, 381, 381, 381, 381, 381, 381, 381, + 381, 381, 381, 0, 381, 381, 381, 381, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 381, 381, 381, 381, 381, + 381, 382, 0, 0, 0, 382, 0, 0, 0, 0, + + 382, 382, 382, 382, 382, 382, 382, 382, 382, 382, + 382, 382, 382, 382, 382, 382, 382, 0, 0, 0, + 0, 0, 382, 383, 0, 0, 0, 0, 383, 383, + 383, 383, 383, 383, 383, 383, 383, 383, 383, 383, + 383, 383, 383, 383, 383, 383, 383, 383, 383, 383, + 383, 383, 383, 383, 383, 383, 383, 383, 383, 383, + 383, 383, 383, 383, 383, 383, 384, 384, 384, 0, + 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, + 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, + 384, 384, 384, 384, 384, 384, 384, 384, 384, 384, + + 384, 384, 384, 384, 384, 384, 384, 384, 384, 385, + 385, 385, 0, 385, 385, 385, 385, 385, 385, 385, + 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + 385, 385, 386, 0, 0, 0, 0, 386, 386, 386, + 386, 386, 386, 386, 386, 386, 386, 386, 386, 386, + 386, 386, 386, 386, 387, 0, 0, 0, 0, 387, + 387, 387, 387, 387, 387, 387, 387, 387, 387, 387, + 387, 387, 387, 387, 387, 387, 388, 388, 388, 388, + + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, + 388, 388, 388, 388, 388, 388, 388, 388, 388, 389, + 389, 389, 389, 389, 389, 389, 389, 389, 389, 389, + 389, 389, 389, 389, 389, 389, 389, 389, 389, 389, + 389, 389, 389, 389, 389, 389, 389, 389, 389, 389, + 389, 389, 389, 389, 389, 389, 389, 389, 389, 389, + 389, 389, 390, 390, 390, 0, 390, 390, 390, 390, + 390, 390, 390, 390, 390, 390, 390, 390, 390, 390, + + 390, 390, 390, 390, 390, 390, 390, 390, 390, 390, + 390, 390, 390, 390, 390, 390, 390, 390, 390, 390, + 390, 390, 0, 390, 390, 391, 391, 391, 391, 391, + 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 391, 392, 392, + 392, 0, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 392, 392, 392, 392, 392, 392, 392, 392, 392, + 392, 393, 393, 393, 393, 393, 393, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 393, 394, 394, 394, 394, 394, 394, + 394, 394, 394, 394, 394, 394, 394, 394, 394, 394, + 394, 394, 394, 394, 394, 394, 394, 394, 394, 394, + 394, 394, 394, 394, 394, 394, 394, 394, 394, 394, + 394, 394, 394, 394, 394, 394, 394, 395, 395, 395, + 395, 395, 395, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 395, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 396, 396, 396, 396, 396, 396, 396, + 396, 396, 396, 397, 0, 0, 0, 397, 0, 0, + 0, 0, 397, 397, 397, 397, 397, 397, 397, 397, + 397, 397, 397, 397, 397, 397, 397, 397, 397, 0, + 0, 0, 0, 0, 397, 398, 0, 0, 0, 398, + 0, 0, 0, 0, 398, 398, 398, 398, 398, 398, + 398, 398, 398, 398, 398, 398, 398, 398, 398, 398, + 398, 0, 0, 0, 0, 0, 398, 399, 399, 399, + 399, 399, 399, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, + 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 400, 400, 400, 400, 400, 400, 400, + 400, 400, 400, 401, 401, 401, 401, 401, 401, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 401, 402, 402, 402, 0, + 402, 402, 402, 402, 402, 402, 402, 402, 402, 402, + 402, 402, 402, 402, 402, 402, 402, 402, 402, 402, + + 402, 402, 402, 402, 402, 402, 402, 402, 402, 402, + 402, 402, 402, 402, 402, 402, 402, 402, 402, 403, + 403, 403, 403, 403, 403, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 403, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340, 340, 340, 340, 340, + 340, 340, 340, 340, 340, 340 + } ; + +static yy_ste_type yy_last_accepting_state; +static YY_CHAR *yy_last_accepting_cpos; + +/* the intent behind this definition is that it'll catch + * any uses of REJECT which flex missed + */ +#define REJECT reject_used_but_not_detected +#define yymore() yymrused_but_not_detected +#define YY_MORE_ADJ 0 + +/* these variables are all declared out here so that section 3 code can + * manipulate them + */ +/* points to current character in buffer */ +static YY_CHAR *yy_c_buf_p = (YY_CHAR *) 0; +static int yy_init = 1; /* whether we need to initialize */ +static int yy_start = 0; /* start state number */ + +/* flag which is used to allow yywrap()'s to do buffer switches + * instead of setting up a fresh yyin. A bit of a hack ... + */ +static int yy_did_buffer_switch_on_eof; + +static yy_ste_type yy_get_previous_state YY_PROTO(( void )); +static yy_ste_type yy_try_NUL_trans YY_PROTO(( yy_ste_type current_state )); +static int yy_get_next_buffer YY_PROTO(( void )); +static void yyunput YY_PROTO(( YY_CHAR c, YY_CHAR *buf_ptr )); +void yyrestrt YY_PROTO(( FILE *input_file )); +void yyswtobf YY_PROTO(( YY_BUFFER_STATE new_buffer )); +void yyldbfst YY_PROTO(( void )); +YY_BUFFER_STATE yycrbffr YY_PROTO(( FILE *file, int size )); +void yydlbffr YY_PROTO(( YY_BUFFER_STATE b )); +void yyinbffr YY_PROTO(( YY_BUFFER_STATE b, FILE *file )); + +#define yy_new_buffer yycrbffr + +#ifdef __cplusplus +static int yyinput YY_PROTO(( void )); +#else +static int input YY_PROTO(( void )); +#endif + +YY_DECL + { + register yy_ste_type yy_current_state; + register YY_CHAR *yy_cp, *yy_bp; + register int yy_act; + + + static int bracelevel, didadef; + int i, indented_code, checking_used, new_xlation; + int doing_codeblock = false; + Char nmdef[MAXLINE], myesc(); + + + if ( yy_init ) + { + YY_USER_INIT; + + if ( ! yy_start ) + yy_start = 1; /* first start state */ + + if ( ! yyin ) + yyin = stdin; + + if ( ! yyout ) + yyout = stdout; + + if ( yy_current_buffer ) + yyinbffr( yy_current_buffer, yyin ); + else + yy_current_buffer = yycrbffr( yyin, YY_BUF_SIZE ); + + yyldbfst(); + + yy_init = 0; + } + + while ( 1 ) /* loops until end-of-file is reached */ + { + yy_cp = yy_c_buf_p; + + /* support of yytext */ + *yy_cp = yy_hold_char; + + /* yy_bp points to the position in yy_ch_buf of the start of the + * current run. + */ + yy_bp = yy_cp; + + yy_current_state = yy_start; + if ( yy_bp[-1] == '\n' ) + ++yy_current_state; +yy_match: + do + { + register YY_CHAR yy_c = yy_ec[*yy_cp]; + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while(yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state) + { + yy_current_state = yy_def[yy_current_state]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + ++yy_cp; + } + while ( yy_current_state != 340 ); + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + +yy_find_action: + yy_act = yy_accept[yy_current_state]; + + YY_DO_BEFORE_ACTION; + YY_USER_ACTION; + +do_action: /* this label is used only to access EOF actions */ + + + switch ( yy_act ) + { + case 0: /* must backtrack */ + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = yy_hold_char; + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + goto yy_find_action; + +case 1: +# line 90 "<stdin>" +indented_code = true; BEGIN(CODEBLOCK); + YY_BREAK +case 2: +# line 91 "<stdin>" +++linenum; /* treat as a comment */ + YY_BREAK +case 3: +# line 92 "<stdin>" +ECHO; BEGIN(C_COMMENT); + YY_BREAK +case 4: +# line 93 "<stdin>" +return ( SCDECL ); + YY_BREAK +case 5: +# line 94 "<stdin>" +return ( XSCDECL ); + YY_BREAK +case 6: +# line 95 "<stdin>" +{ + ++linenum; + lndirout( stdout ); + indented_code = false; + BEGIN(CODEBLOCK); + } + YY_BREAK +case 7: +# line 102 "<stdin>" +return ( WHITESPACE ); + YY_BREAK +case 8: +# line 104 "<stdin>" +{ + sectnum = 2; + lndirout( stdout ); + BEGIN(SECT2PROLOG); + return ( SECTEND ); + } + YY_BREAK +case 9: +# line 111 "<stdin>" +{ + pptmsg( "warning - %%used/%%unused have been deprecated" ); + checking_used = REALLY_USED; BEGIN(USED_LIST); + } + YY_BREAK +case 10: +# line 115 "<stdin>" +{ + checking_used = REALLY_NOT_USED; BEGIN(USED_LIST); + pptmsg( "warning - %%used/%%unused have been deprecated" ); + checking_used = REALLY_NOT_USED; BEGIN(USED_LIST); + } + YY_BREAK +case 11: +# line 122 "<stdin>" +{ +#ifdef NOTDEF + fprintf( stderr, + "old-style lex command at line %d ignored:\n\t%s", + linenum, yytext ); +#endif + ++linenum; + } + YY_BREAK +case 12: +# line 131 "<stdin>" +/* ignore old lex directive */ + YY_BREAK +case 13: +# line 133 "<stdin>" +{ + ++linenum; + xlation = + (int *) malloc( sizeof( int ) * (unsigned) csize ); + + if ( ! xlation ) + flxfatal( + "dynamic memory failure building %t table" ); + + for ( i = 0; i < csize; ++i ) + xlation[i] = 0; + + nuxlatns = 0; + + BEGIN(XLATION); + } + YY_BREAK +case 14: +# line 150 "<stdin>" +synerr( "unrecognized '%' directive" ); + YY_BREAK +case 15: +# line 152 "<stdin>" +{ + (void) strcpy( nmstr, (char *) yytext ); + didadef = false; + BEGIN(PICKUPDEF); + } + YY_BREAK +case 16: +# line 158 "<stdin>" +RETURNNAME; + YY_BREAK +case 17: +# line 159 "<stdin>" +++linenum; /* allows blank lines in section 1 */ + YY_BREAK +case 18: +# line 160 "<stdin>" +++linenum; return ( '\n' ); + YY_BREAK +case 19: +# line 161 "<stdin>" +synerr( "illegal character" ); BEGIN(RECOVER); + YY_BREAK +case 20: +# line 164 "<stdin>" +ECHO; BEGIN(INITIAL); + YY_BREAK +case 21: +# line 165 "<stdin>" +++linenum; ECHO; BEGIN(INITIAL); + YY_BREAK +case 22: +# line 166 "<stdin>" +ECHO; + YY_BREAK +case 23: +# line 167 "<stdin>" +ECHO; + YY_BREAK +case 24: +# line 168 "<stdin>" +++linenum; ECHO; + YY_BREAK +case 25: +# line 171 "<stdin>" +++linenum; BEGIN(INITIAL); + YY_BREAK +case 26: +# line 172 "<stdin>" +ECHO; CHECK_REJECT(yytext); + YY_BREAK +case 27: +# line 173 "<stdin>" +ECHO; CHECK_YYMORE(yytext); + YY_BREAK +case 28: +# line 174 "<stdin>" +ECHO; + YY_BREAK +case 29: +# line 175 "<stdin>" +{ + ++linenum; + ECHO; + if ( indented_code ) + BEGIN(INITIAL); + } + YY_BREAK +case 30: +# line 183 "<stdin>" +/* separates name and definition */ + YY_BREAK +case 31: +# line 185 "<stdin>" +{ + (void) strcpy( (char *) nmdef, (char *) yytext ); + + for ( i = strlen( (char *) nmdef ) - 1; + i >= 0 && + nmdef[i] == ' ' || nmdef[i] == '\t'; + --i ) + ; + + nmdef[i + 1] = '\0'; + + ndinstal( nmstr, nmdef ); + didadef = true; + } + YY_BREAK +case 32: +# line 200 "<stdin>" +{ + if ( ! didadef ) + synerr( "incomplete name definition" ); + BEGIN(INITIAL); + ++linenum; + } + YY_BREAK +case 33: +# line 207 "<stdin>" +++linenum; BEGIN(INITIAL); RETURNNAME; + YY_BREAK +case 34: +# line 210 "<stdin>" +++linenum; BEGIN(INITIAL); + YY_BREAK +case 35: +# line 211 "<stdin>" + + YY_BREAK +case 36: +# line 212 "<stdin>" +{ + if ( allupper( yytext ) ) + rjctused = checking_used; + else + synerr( "unrecognized %used/%unused construct" ); + } + YY_BREAK +case 37: +# line 218 "<stdin>" +{ + if ( alllower( yytext ) ) + yymrreus = checking_used; + else + synerr( "unrecognized %used/%unused construct" ); + } + YY_BREAK +case 38: +# line 224 "<stdin>" +synerr( "unrecognized %used/%unused construct" ); + YY_BREAK +case 39: +# line 227 "<stdin>" +++linenum; BEGIN(INITIAL); + YY_BREAK +case 40: +# line 228 "<stdin>" +++nuxlatns; new_xlation = true; + YY_BREAK +case 41: +# line 229 "<stdin>" +synerr( "bad row in translation table" ); + YY_BREAK +case 42: +# line 230 "<stdin>" +/* ignore whitespace */ + YY_BREAK +case 43: +# line 232 "<stdin>" +{ + xlation[myesc( yytext )] = + (new_xlation ? nuxlatns : -nuxlatns); + new_xlation = false; + } + YY_BREAK +case 44: +# line 237 "<stdin>" +{ + xlation[yytext[0]] = + (new_xlation ? nuxlatns : -nuxlatns); + new_xlation = false; + } + YY_BREAK +case 45: +# line 243 "<stdin>" +++linenum; + YY_BREAK +case 46: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp -= 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 246 "<stdin>" +{ + ++linenum; + ACTION_ECHO; + MARK_END_OF_PROLOG; + BEGIN(SECT2); + } + YY_BREAK +case 47: +# line 253 "<stdin>" +++linenum; ACTION_ECHO; + YY_BREAK +case YY_STATE_EOF(SECT2PROLOG): +# line 255 "<stdin>" +MARK_END_OF_PROLOG; yyterminate(); + YY_BREAK +case 49: +# line 257 "<stdin>" +++linenum; /* allow blank lines in section 2 */ + YY_BREAK +case 50: +# line 259 "<stdin>" +{ + indented_code = (yytext[0] != '%'); + doing_codeblock = true; + bracelevel = 1; + + if ( indented_code ) + ACTION_ECHO; + + BEGIN(CODEBLOCK_2); + } + YY_BREAK +case 51: +# line 270 "<stdin>" +BEGIN(SC); return ( '<' ); + YY_BREAK +case 52: +# line 271 "<stdin>" +return ( '^' ); + YY_BREAK +case 53: +# line 272 "<stdin>" +BEGIN(QUOTE); return ( '"' ); + YY_BREAK +case 54: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 273 "<stdin>" +BEGIN(NUM); return ( '{' ); + YY_BREAK +case 55: +# line 274 "<stdin>" +BEGIN(BRACEERROR); + YY_BREAK +case 56: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 275 "<stdin>" +return ( '$' ); + YY_BREAK +case 57: +# line 277 "<stdin>" +{ + bracelevel = 1; + BEGIN(PERCENT_BRACE_ACTION); + return ( '\n' ); + } + YY_BREAK +case 58: +# line 282 "<stdin>" +cntdactn = true; ++linenum; return ( '\n' ); + YY_BREAK +case 59: +# line 284 "<stdin>" +{ + /* this rule is separate from the one below because + * otherwise we get variable trailing context, so + * we can't build the scanner using -{f,F} + */ + bracelevel = 0; + cntdactn = false; + BEGIN(ACTION); + return ( '\n' ); + } + YY_BREAK +case 60: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp -= 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 295 "<stdin>" +{ + bracelevel = 0; + cntdactn = false; + BEGIN(ACTION); + return ( '\n' ); + } + YY_BREAK +case 61: +# line 302 "<stdin>" +++linenum; return ( '\n' ); + YY_BREAK +case 62: +# line 304 "<stdin>" +return ( EOF_OP ); + YY_BREAK +case 63: +# line 306 "<stdin>" +{ + sectnum = 3; + BEGIN(SECT3); + return ( EOF ); /* to stop the parser */ + } + YY_BREAK +case 64: +# line 312 "<stdin>" +{ + int cclval; + + (void) strcpy( nmstr, (char *) yytext ); + + /* check to see if we've already encountered this ccl */ + if ( (cclval = ccllookp( (Char *) nmstr )) ) + { + yylval = cclval; + ++cclreuse; + return ( PREVCCL ); + } + else + { + /* we fudge a bit. We know that this ccl will + * soon be numbered as lastccl + 1 by cclinit + */ + cclnstal( (Char *) nmstr, lastccl + 1 ); + + /* push back everything but the leading bracket + * so the ccl can be rescanned + */ + PUT_BACK_STRING((Char *) nmstr, 1); + + BEGIN(FIRSTCCL); + return ( '[' ); + } + } + YY_BREAK +case 65: +# line 341 "<stdin>" +{ + register Char *nmdefptr; + Char *ndlookup(); + + (void) strcpy( nmstr, (char *) yytext ); + nmstr[yyleng - 1] = '\0'; /* chop trailing brace */ + + /* lookup from "nmstr + 1" to chop leading brace */ + if ( ! (nmdefptr = ndlookup( nmstr + 1 )) ) + synerr( "undefined {name}" ); + + else + { /* push back name surrounded by ()'s */ + unput(')'); + PUT_BACK_STRING(nmdefptr, 0); + unput('('); + } + } + YY_BREAK +case 66: +# line 360 "<stdin>" +return ( yytext[0] ); + YY_BREAK +case 67: +# line 361 "<stdin>" +RETURNCHAR; + YY_BREAK +case 68: +# line 362 "<stdin>" +++linenum; return ( '\n' ); + YY_BREAK +case 69: +# line 365 "<stdin>" +return ( ',' ); + YY_BREAK +case 70: +# line 366 "<stdin>" +BEGIN(SECT2); return ( '>' ); + YY_BREAK +case 71: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 367 "<stdin>" +BEGIN(CARETISBOL); return ( '>' ); + YY_BREAK +case 72: +# line 368 "<stdin>" +RETURNNAME; + YY_BREAK +case 73: +# line 369 "<stdin>" +synerr( "bad start condition name" ); + YY_BREAK +case 74: +# line 371 "<stdin>" +BEGIN(SECT2); return ( '^' ); + YY_BREAK +case 75: +# line 374 "<stdin>" +RETURNCHAR; + YY_BREAK +case 76: +# line 375 "<stdin>" +BEGIN(SECT2); return ( '"' ); + YY_BREAK +case 77: +# line 377 "<stdin>" +{ + synerr( "missing quote" ); + BEGIN(SECT2); + ++linenum; + return ( '"' ); + } + YY_BREAK +case 78: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 385 "<stdin>" +BEGIN(CCL); return ( '^' ); + YY_BREAK +case 79: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 386 "<stdin>" +return ( '^' ); + YY_BREAK +case 80: +# line 387 "<stdin>" +BEGIN(CCL); yylval = '-'; return ( CHAR ); + YY_BREAK +case 81: +# line 388 "<stdin>" +BEGIN(CCL); RETURNCHAR; + YY_BREAK +case 82: +*yy_cp = yy_hold_char; /* undo effects of setting up yytext */ +yy_c_buf_p = yy_cp = yy_bp + 1; +YY_DO_BEFORE_ACTION; /* set up yytext again */ +# line 390 "<stdin>" +return ( '-' ); + YY_BREAK +case 83: +# line 391 "<stdin>" +RETURNCHAR; + YY_BREAK +case 84: +# line 392 "<stdin>" +BEGIN(SECT2); return ( ']' ); + YY_BREAK +case 85: +# line 395 "<stdin>" +{ + yylval = myctoi( yytext ); + return ( NUMBER ); + } + YY_BREAK +case 86: +# line 400 "<stdin>" +return ( ',' ); + YY_BREAK +case 87: +# line 401 "<stdin>" +BEGIN(SECT2); return ( '}' ); + YY_BREAK +case 88: +# line 403 "<stdin>" +{ + synerr( "bad character inside {}'s" ); + BEGIN(SECT2); + return ( '}' ); + } + YY_BREAK +case 89: +# line 409 "<stdin>" +{ + synerr( "missing }" ); + BEGIN(SECT2); + ++linenum; + return ( '}' ); + } + YY_BREAK +case 90: +# line 417 "<stdin>" +synerr( "bad name in {}'s" ); BEGIN(SECT2); + YY_BREAK +case 91: +# line 418 "<stdin>" +synerr( "missing }" ); ++linenum; BEGIN(SECT2); + YY_BREAK +case 92: +# line 421 "<stdin>" +bracelevel = 0; + YY_BREAK +case 93: +# line 422 "<stdin>" +{ + ACTION_ECHO; + CHECK_REJECT(yytext); + } + YY_BREAK +case 94: +# line 426 "<stdin>" +{ + ACTION_ECHO; + CHECK_YYMORE(yytext); + } + YY_BREAK +case 95: +# line 430 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 96: +# line 431 "<stdin>" +{ + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 || + (doing_codeblock && indented_code) ) + { + if ( ! doing_codeblock ) + fputs( "\tYY_BREAK\n", tmpactfl ); + + doing_codeblock = false; + BEGIN(SECT2); + } + } + YY_BREAK + /* Reject and YYmore() are checked for above, in PERCENT_BRACE_ACTION */ +case 97: +# line 447 "<stdin>" +ACTION_ECHO; ++bracelevel; + YY_BREAK +case 98: +# line 448 "<stdin>" +ACTION_ECHO; --bracelevel; + YY_BREAK +case 99: +# line 449 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 100: +# line 450 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 101: +# line 451 "<stdin>" +ACTION_ECHO; BEGIN(ACTION_COMMENT); + YY_BREAK +case 102: +# line 452 "<stdin>" +ACTION_ECHO; /* character constant */ + YY_BREAK +case 103: +# line 453 "<stdin>" +ACTION_ECHO; BEGIN(ACTION_STRING); + YY_BREAK +case 104: +# line 454 "<stdin>" +{ + ++linenum; + ACTION_ECHO; + if ( bracelevel == 0 ) + { + fputs( "\tYY_BREAK\n", tmpactfl ); + BEGIN(SECT2); + } + } + YY_BREAK +case 105: +# line 463 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 106: +# line 465 "<stdin>" +ACTION_ECHO; BEGIN(ACTION); + YY_BREAK +case 107: +# line 466 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 108: +# line 467 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 109: +# line 468 "<stdin>" +++linenum; ACTION_ECHO; + YY_BREAK +case 110: +# line 469 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 111: +# line 471 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 112: +# line 472 "<stdin>" +ACTION_ECHO; + YY_BREAK +case 113: +# line 473 "<stdin>" +++linenum; ACTION_ECHO; + YY_BREAK +case 114: +# line 474 "<stdin>" +ACTION_ECHO; BEGIN(ACTION); + YY_BREAK +case 115: +# line 475 "<stdin>" +ACTION_ECHO; + YY_BREAK +case YY_STATE_EOF(ACTION): +case YY_STATE_EOF(ACTION_COMMENT): +case YY_STATE_EOF(ACTION_STRING): +# line 477 "<stdin>" +{ + synerr( "EOF encountered inside an action" ); + yyterminate(); + } + YY_BREAK +case 117: +# line 483 "<stdin>" +{ + yylval = myesc( yytext ); + return ( CHAR ); + } + YY_BREAK +case 118: +# line 488 "<stdin>" +{ + yylval = myesc( yytext ); + BEGIN(CCL); + return ( CHAR ); + } + YY_BREAK +case 119: +# line 495 "<stdin>" +ECHO; + YY_BREAK +case 120: +# line 496 "<stdin>" +YY_FATAL_ERROR( "flex scanner jammed" ); + YY_BREAK +case YY_STATE_EOF(INITIAL): +case YY_STATE_EOF(SECT2): +case YY_STATE_EOF(SECT3): +case YY_STATE_EOF(CODEBLOCK): +case YY_STATE_EOF(PICKUPDEF): +case YY_STATE_EOF(SC): +case YY_STATE_EOF(CARETISBOL): +case YY_STATE_EOF(NUM): +case YY_STATE_EOF(QUOTE): +case YY_STATE_EOF(FIRSTCCL): +case YY_STATE_EOF(CCL): +case YY_STATE_EOF(RECOVER): +case YY_STATE_EOF(BRACEERROR): +case YY_STATE_EOF(C_COMMENT): +case YY_STATE_EOF(PERCENT_BRACE_ACTION): +case YY_STATE_EOF(USED_LIST): +case YY_STATE_EOF(CODEBLOCK_2): +case YY_STATE_EOF(XLATION): + yyterminate(); + + case YY_END_OF_BUFFER: + { + /* amount of text matched not including the EOB char */ + int yy_amount_of_matched_text = yy_cp - yytext - 1; + + /* undo the effects of YY_DO_BEFORE_ACTION */ + *yy_cp = yy_hold_char; + + /* note that here we test for yy_c_buf_p "<=" to the position + * of the first EOB in the buffer, since yy_c_buf_p will + * already have been incremented past the NUL character + * (since all states make transitions on EOB to the end- + * of-buffer state). Contrast this with the test in yyinput(). + */ + if ( yy_c_buf_p <= &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + /* this was really a NUL */ + { + yy_ste_type yy_next_state; + + yy_c_buf_p = yytext + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + /* okay, we're now positioned to make the + * NUL transition. We couldn't have + * yy_get_previous_state() go ahead and do it + * for us because it doesn't know how to deal + * with the possibility of jamming (and we + * don't want to build jamming into it because + * then it will run more slowly) + */ + + yy_next_state = yy_try_NUL_trans( yy_current_state ); + + yy_bp = yytext + YY_MORE_ADJ; + + if ( yy_next_state ) + { + /* consume the NUL */ + yy_cp = ++yy_c_buf_p; + yy_current_state = yy_next_state; + goto yy_match; + } + + else + { + yy_cp = yy_last_accepting_cpos; + yy_current_state = yy_last_accepting_state; + goto yy_find_action; + } + } + + else switch ( yy_get_next_buffer() ) + { + case EOB_ACT_END_OF_FILE: + { + yy_did_buffer_switch_on_eof = 0; + + if ( yywrap() ) + { + /* note: because we've taken care in + * yy_get_next_buffer() to have set up yytext, + * we can now set up yy_c_buf_p so that if some + * total hoser (like flex itself) wants + * to call the scanner after we return the + * YY_NULL, it'll still work - another YY_NULL + * will get returned. + */ + yy_c_buf_p = yytext + YY_MORE_ADJ; + + yy_act = YY_STATE_EOF((yy_start - 1) / 2); + goto do_action; + } + + else + { + if ( ! yy_did_buffer_switch_on_eof ) + YY_NEW_FILE; + } + } + break; + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext + yy_amount_of_matched_text; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext + YY_MORE_ADJ; + goto yy_match; + + case EOB_ACT_LAST_MATCH: + yy_c_buf_p = + &yy_current_buffer->yy_ch_buf[yy_n_chars]; + + yy_current_state = yy_get_previous_state(); + + yy_cp = yy_c_buf_p; + yy_bp = yytext + YY_MORE_ADJ; + goto yy_find_action; + } + break; + } + + default: +#ifdef FLEX_DEBUG + printf( "action # %d\n", yy_act ); +#endif + YY_FATAL_ERROR( + "fatal flex scanner internal error--no action found" ); + } + } + } + + +/* yy_get_next_buffer - try to read in a new buffer + * + * synopsis + * int yy_get_next_buffer(); + * + * returns a code representing an action + * EOB_ACT_LAST_MATCH - + * EOB_ACT_CONTINUE_SCAN - continue scanning from current position + * EOB_ACT_END_OF_FILE - end of file + */ + +static int yy_get_next_buffer() + + { + register YY_CHAR *dest = yy_current_buffer->yy_ch_buf; + register YY_CHAR *source = yytext - 1; /* copy prev. char, too */ + register int number_to_move, i; + int ret_val; + + if ( yy_c_buf_p > &yy_current_buffer->yy_ch_buf[yy_n_chars + 1] ) + YY_FATAL_ERROR( + "fatal flex scanner internal error--end of buffer missed" ); + + /* try to read more data */ + + /* first move last chars to start of buffer */ + number_to_move = yy_c_buf_p - yytext; + + for ( i = 0; i < number_to_move; ++i ) + *(dest++) = *(source++); + + if ( yy_current_buffer->yy_eof_status != EOF_NOT_SEEN ) + /* don't do the read, it's not guaranteed to return an EOF, + * just force an EOF + */ + yy_n_chars = 0; + + else + { + int num_to_read = yy_current_buffer->yy_buf_size - number_to_move - 1; + + if ( num_to_read > YY_READ_BUF_SIZE ) + num_to_read = YY_READ_BUF_SIZE; + + else if ( num_to_read <= 0 ) + YY_FATAL_ERROR( "fatal error - scanner input buffer overflow" ); + + /* read in more data */ + YY_INPUT( (&yy_current_buffer->yy_ch_buf[number_to_move]), + yy_n_chars, num_to_read ); + } + + if ( yy_n_chars == 0 ) + { + if ( number_to_move == 1 ) + { + ret_val = EOB_ACT_END_OF_FILE; + yy_current_buffer->yy_eof_status = EOF_DONE; + } + + else + { + ret_val = EOB_ACT_LAST_MATCH; + yy_current_buffer->yy_eof_status = EOF_PENDING; + } + } + + else + ret_val = EOB_ACT_CONTINUE_SCAN; + + yy_n_chars += number_to_move; + yy_current_buffer->yy_ch_buf[yy_n_chars] = YY_END_OF_BUFFER_CHAR; + yy_current_buffer->yy_ch_buf[yy_n_chars + 1] = YY_END_OF_BUFFER_CHAR; + + /* yytext begins at the second character in yy_ch_buf; the first + * character is the one which preceded it before reading in the latest + * buffer; it needs to be kept around in case it's a newline, so + * yy_get_previous_state() will have with '^' rules active + */ + + yytext = &yy_current_buffer->yy_ch_buf[1]; + + return ( ret_val ); + } + + +/* yy_get_previous_state - get the state just before the EOB char was reached + * + * synopsis + * yy_ste_type yy_get_previous_state(); + */ + +static yy_ste_type yy_get_previous_state() + + { + register yy_ste_type yy_current_state; + register YY_CHAR *yy_cp; + + register YY_CHAR *yy_bp = yytext; + + yy_current_state = yy_start; + if ( yy_bp[-1] == '\n' ) + ++yy_current_state; + + for ( yy_cp = yytext + YY_MORE_ADJ; yy_cp < yy_c_buf_p; ++yy_cp ) + { + register YY_CHAR yy_c = (*yy_cp ? yy_ec[*yy_cp] : 1); + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = yy_def[yy_current_state]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + } + + return ( yy_current_state ); + } + + +/* yy_try_NUL_trans - try to make a transition on the NUL character + * + * synopsis + * next_state = yy_try_NUL_trans( current_state ); + */ + +#ifdef YY_USE_PROTOS +static yy_ste_type yy_try_NUL_trans( register yy_ste_type yy_current_state ) +#else +static yy_ste_type yy_try_NUL_trans( yy_current_state ) +register yy_ste_type yy_current_state; +#endif + + { + register int yy_is_jam; + register YY_CHAR *yy_cp = yy_c_buf_p; + + register YY_CHAR yy_c = 1; + if ( yy_accept[yy_current_state] ) + { + yy_last_accepting_state = yy_current_state; + yy_last_accepting_cpos = yy_cp; + } + while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) + { + yy_current_state = yy_def[yy_current_state]; + } + yy_current_state = yy_nxt[yy_base[yy_current_state] + yy_c]; + yy_is_jam = (yy_current_state == 340); + + return ( yy_is_jam ? 0 : yy_current_state ); + } + + +#ifdef YY_USE_PROTOS +static void yyunput( YY_CHAR c, register YY_CHAR *yy_bp ) +#else +static void yyunput( c, yy_bp ) +YY_CHAR c; +register YY_CHAR *yy_bp; +#endif + + { + register YY_CHAR *yy_cp = yy_c_buf_p; + + /* undo effects of setting up yytext */ + *yy_cp = yy_hold_char; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + { /* need to shift things up to make room */ + register int number_to_move = yy_n_chars + 2; /* +2 for EOB chars */ + register YY_CHAR *dest = + &yy_current_buffer->yy_ch_buf[yy_current_buffer->yy_buf_size + 2]; + register YY_CHAR *source = + &yy_current_buffer->yy_ch_buf[number_to_move]; + + while ( source > yy_current_buffer->yy_ch_buf ) + *--dest = *--source; + + yy_cp += dest - source; + yy_bp += dest - source; + yy_n_chars = yy_current_buffer->yy_buf_size; + + if ( yy_cp < yy_current_buffer->yy_ch_buf + 2 ) + YY_FATAL_ERROR( "flex scanner push-back overflow" ); + } + + if ( yy_cp > yy_bp && yy_cp[-1] == '\n' ) + yy_cp[-2] = '\n'; + + *--yy_cp = c; + + /* note: the formal parameter *must* be called "yy_bp" for this + * macro to now work correctly + */ + YY_DO_BEFORE_ACTION; /* set up yytext again */ + } + + +#ifdef __cplusplus +static int yyinput() +#else +static int input() +#endif + + { + int c; + YY_CHAR *yy_cp = yy_c_buf_p; + + *yy_cp = yy_hold_char; + + if ( *yy_c_buf_p == YY_END_OF_BUFFER_CHAR ) + { + /* yy_c_buf_p now points to the character we want to return. + * If this occurs *before* the EOB characters, then it's a + * valid NUL; if not, then we've hit the end of the buffer. + */ + if ( yy_c_buf_p < &yy_current_buffer->yy_ch_buf[yy_n_chars] ) + /* this was really a NUL */ + *yy_c_buf_p = '\0'; + + else + { /* need more input */ + yytext = yy_c_buf_p; + ++yy_c_buf_p; + + switch ( yy_get_next_buffer() ) + { + case EOB_ACT_END_OF_FILE: + { + if ( yywrap() ) + { + yy_c_buf_p = yytext + YY_MORE_ADJ; + return ( EOF ); + } + + YY_NEW_FILE; + +#ifdef __cplusplus + return ( yyinput() ); +#else + return ( input() ); +#endif + } + break; + + case EOB_ACT_CONTINUE_SCAN: + yy_c_buf_p = yytext + YY_MORE_ADJ; + break; + + case EOB_ACT_LAST_MATCH: +#ifdef __cplusplus + YY_FATAL_ERROR( "unexpected last match in yyinput()" ); +#else + YY_FATAL_ERROR( "unexpected last match in input()" ); +#endif + } + } + } + + c = *yy_c_buf_p; + yy_hold_char = *++yy_c_buf_p; + + return ( c ); + } + + +#ifdef YY_USE_PROTOS +void yyrestrt( FILE *input_file ) +#else +void yyrestrt( input_file ) +FILE *input_file; +#endif + + { + yyinbffr( yy_current_buffer, input_file ); + yyldbfst(); + } + + +#ifdef YY_USE_PROTOS +void yyswtobf( YY_BUFFER_STATE new_buffer ) +#else +void yyswtobf( new_buffer ) +YY_BUFFER_STATE new_buffer; +#endif + + { + if ( yy_current_buffer == new_buffer ) + return; + + if ( yy_current_buffer ) + { + /* flush out information for old buffer */ + *yy_c_buf_p = yy_hold_char; + yy_current_buffer->yy_buf_pos = yy_c_buf_p; + yy_current_buffer->yy_n_chars = yy_n_chars; + } + + yy_current_buffer = new_buffer; + yyldbfst(); + + /* we don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yy_did_buffer_switch_on_eof = 1; + } + + +#ifdef YY_USE_PROTOS +void yyldbfst( void ) +#else +void yyldbfst() +#endif + + { + yy_n_chars = yy_current_buffer->yy_n_chars; + yytext = yy_c_buf_p = yy_current_buffer->yy_buf_pos; + yyin = yy_current_buffer->yy_input_file; + yy_hold_char = *yy_c_buf_p; + } + + +#ifdef YY_USE_PROTOS +YY_BUFFER_STATE yycrbffr( FILE *file, int size ) +#else +YY_BUFFER_STATE yycrbffr( file, size ) +FILE *file; +int size; +#endif + + { + YY_BUFFER_STATE b; + + b = (YY_BUFFER_STATE) malloc( sizeof( struct yy_buffer_state ) ); + + if ( ! b ) + YY_FATAL_ERROR( "out of dynamic memory in yycrbffr()" ); + + b->yy_buf_size = size; + + /* yy_ch_buf has to be 2 characters longer than the size given because + * we need to put in 2 end-of-buffer characters. + */ + b->yy_ch_buf = (YY_CHAR *) malloc( (unsigned) (b->yy_buf_size + 2) ); + + if ( ! b->yy_ch_buf ) + YY_FATAL_ERROR( "out of dynamic memory in yycrbffr()" ); + + yyinbffr( b, file ); + + return ( b ); + } + + +#ifdef YY_USE_PROTOS +void yydlbffr( YY_BUFFER_STATE b ) +#else +void yydlbffr( b ) +YY_BUFFER_STATE b; +#endif + + { + if ( b == yy_current_buffer ) + yy_current_buffer = (YY_BUFFER_STATE) 0; + + free( (char *) b->yy_ch_buf ); + free( (char *) b ); + } + + +#ifdef YY_USE_PROTOS +void yyinbffr( YY_BUFFER_STATE b, FILE *file ) +#else +void yyinbffr( b, file ) +YY_BUFFER_STATE b; +FILE *file; +#endif + + { + b->yy_input_file = file; + + /* we put in the '\n' and start reading from [1] so that an + * initial match-at-newline will be true. + */ + + b->yy_ch_buf[0] = '\n'; + b->yy_n_chars = 1; + + /* we always need two end-of-buffer characters. The first causes + * a transition to the end-of-buffer state. The second causes + * a jam in that state. + */ + b->yy_ch_buf[1] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[2] = YY_END_OF_BUFFER_CHAR; + + b->yy_buf_pos = &b->yy_ch_buf[1]; + + b->yy_eof_status = EOF_NOT_SEEN; + } +# line 496 "<stdin>" + + + +int yywrap() + + { + if ( --ninfiles > 0 ) + { + stinpfle( *++inp_fles ); + return ( 0 ); + } + + else + return ( 1 ); + } + + +/* stinpfle - open the given file (if NULL, stdin) for scanning */ + +void stinpfle( file ) +char *file; + + { + if ( file ) + { + infilnam = file; + yyin = fopen( infilnam, "r" ); + + if ( yyin == NULL ) + lerrsf( "can't open %s", file ); + } + + else + { + yyin = stdin; + infilnam = "<stdin>"; + } + } diff --git a/MISC/MVS/unfixit.l b/MISC/MVS/unfixit.l new file mode 100644 index 0000000..3f8dddc --- /dev/null +++ b/MISC/MVS/unfixit.l @@ -0,0 +1,141 @@ +%{ +/* unfixit.l - convert shortened external names to names back to their + * original names. (See fixit.l) + */ + +/* + * This program is included to satisfy "dumb" compilers/linkers which + * do not know about externals of names longer than 8 bytes. + * + * Steven W. Layten + * Chemical Abstracts Service + * PO BOX 3012 + * Columbus, OH 43210 + */ +%} +%% +"actfilnm" printf("action_file_name"); +"actnout" printf("action_out"); +"addacpt" printf("add_accept"); +"alllower" printf("all_lower"); +"allupper" printf("all_upper"); +"allocarr" printf("allocate_array"); +"asscrule" printf("assoc_rule"); +"bktrkfil" printf("backtrack_file"); +"bktrkrep" printf("backtrack_report"); +"bol_nded" printf("bol_needed"); +"bldeofac" printf("build_eof_action"); +"cclnstal" printf("cclinstal"); +"ccllookp" printf("ccllookup"); +"cclnegat" printf("cclnegate"); +"cclsrted" printf("cclsorted"); +"ck4bktrk" printf("check_for_backtracking"); +"cktrlcnt" printf("check_trailing_context"); +"cntdactn" printf("continued_action"); +"copystrn" printf("copy_string"); +"cpunsstr" printf("copy_unsigned_string"); +"cpyrght" printf("copyright"); +"copysngl" printf("copysingl"); +"c_mx_ccl" printf("current_max_ccl_tbl_size"); +"c_mx_dfa" printf("current_max_dfa_size"); +"c_mxdfas" printf("current_max_dfas"); +"curmxrls" printf("current_max_rules"); +"c_mx_scs" printf("current_max_scs"); +"c_mx_tmp" printf("current_max_template_xpairs"); +"c_mx_xpr" printf("current_max_xpairs"); +"c_mxccls" printf("current_maxccls"); +"curr_mns" printf("current_mns"); +"cursttyp" printf("current_state_type"); +"datflush" printf("dataflush"); +"dfacunin" printf("dfaacc_union"); +"do_indnt" printf("do_indent"); +"dmpasrl" printf("dump_associated_rules"); +"dmptrns" printf("dump_transitions"); +"dupmach" printf("dupmachine"); +"ecsfrmxt" printf("ecs_from_xlation"); +"eobstate" printf("end_of_buffer_state"); +"epsclos" printf("epsclosure"); +"expnxtck" printf("expand_nxt_chk"); +"fndtblsp" printf("find_table_space"); +"fnshrule" printf("finish_rule"); +"firstfre" printf("firstfree"); +"firstprt" printf("firstprot"); +"flxgettm" printf("flex_gettime"); +"flxerror" printf("flexerror"); +"flxfatal" printf("flexfatal"); +"fmtptmsg" printf("format_pinpoint_message"); +"gnNULtrn" printf("gen_NUL_trans"); +"gnbktrkg" printf("gen_backtracking"); +"gnbtactn" printf("gen_bt_action"); +"gnfndact" printf("gen_find_action"); +"gnlindir" printf("gen_line_dirs"); +"gnnxcste" printf("gen_next_compressed_state"); +"gnnxmtch" printf("gen_next_match"); +"gnnxtst" printf("gen_next_state"); +"gnstrtst" printf("gen_start_state"); +"hshentry" printf("hash_entry"); +"hshfct" printf("hashfunct"); +"incmxdfa" printf("increase_max_dfas"); +"indput2s" printf("indent_put2s"); +"indputs" printf("indent_puts"); +"infilnam" printf("infilename"); +"inp_fles" printf("input_files"); +"intractv" printf("interactive"); +"lndirout" printf("line_directive_out"); +"lnkmchns" printf("link_machines"); +"lst_cset" printf("list_character_set"); +"maketbls" printf("make_tables"); +"mkbgnorm" printf("mark_beginning_as_normal"); +"mktmplat" printf("mktemplate"); +"nbktrckg" printf("num_backtracking"); +"ninfiles" printf("num_input_files"); +"numraloc" printf("num_reallocs"); +"numrules" printf("num_rules"); +"nuxlatns" printf("num_xlations"); +"numnpair" printf("numsnpairs"); +"outfilnm" printf("output_file_name"); +"peakpair" printf("peakpairs"); +"perf_rep" printf("performance_report"); +"pptmsg" printf("pinpoint_message"); +"plcstate" printf("place_state"); +"prvctdan" printf("previous_continued_action"); +"prtstats" printf("printstats"); +"pgm_name" printf("program_name"); +"prtcomst" printf("protcomst"); +"rdblefrm" printf("readable_form"); +"realrjct" printf("real_reject"); +"rallocar" printf("reallocate_array"); +"rjctused" printf("reject_really_used"); +"rulelnno" printf("rule_linenum"); +"ruletype" printf("rule_type"); +"stinpfle" printf("set_input_file"); +"setupia" printf("set_up_initial_allocations"); +"startime" printf("starttime"); +"ste_type" printf("state_type"); +"symfollo" printf("symfollowset"); +"sympartn" printf("sympartition"); +"syntxerr" printf("syntaxerror"); +"tmpactfl" printf("temp_action_file"); +"todohead" printf("todo_head"); +"todonext" printf("todo_next"); +"trnschar" printf("transchar"); +"trnstout" printf("transition_struct_out"); +"trlcntxt" printf("trlcontxt"); +"vtrailrl" printf("variable_trail_rule"); +"vtrlctrl" printf("variable_trailing_context_rules"); +"varlngth" printf("varlength"); +"yycrbffr" printf("yy_create_buffer"); +"yydlbffr" printf("yy_delete_buffer"); +"yyinbffr" printf("yy_init_buffer"); +"yyldbfst" printf("yy_load_buffer_state"); +"yyswtobf" printf("yy_switch_to_buffer"); +"yyerrflg" printf("yyerrflag"); +"yymrreus" printf("yymore_really_used"); +"yymrused" printf("yymore_used"); +"yyrestrt" printf("yyrestart"); +. ECHO; +%% +main() +{ + yylex(); +} diff --git a/MISC/Macintosh/THINK_C_notes b/MISC/Macintosh/THINK_C_notes new file mode 100644 index 0000000..e99c972 --- /dev/null +++ b/MISC/Macintosh/THINK_C_notes @@ -0,0 +1,100 @@ +Notes on the THINK C version of Flex 2.4.6 +Scott Hofmann 23-JUL-94 +Internet: scotth@visix.com + +The only changes needed to compile Flex 2.4.6 under Symantec C++ 6.0 was +to #include <console.h> in main.c and call ccommand() just before flexinit() +in main(). The notes below are mostly of historical significance only; most +of the workarounds below were to get around restrictions/problems in earlier +versions of THINK C. The only section which still applies is Russell Finn's +description of how to make Flex generate output of type 'KAHL'. Also, 4-byte +ints must be used by any project which uses Flex output. + +If you want to recreate the project, you'll need to add the files +alloca.c and xmalloc.c in this directory. Both files are copylefted; see +the GNU General Public License for details. You will also need to recompile +both the ANSI and unix libraries to use 4 byte ints, and if you want the +files that flex creates to have 'KAHL' as the creator you'll need to apply +Russell Finn's patch. + +Notes on the THINK C version of Flex 2.3.7 +Jonas Barklund, 25-JAN-92 +Internet: jonas@csd.uu.se + +I have merged the sources for Flex version 2.3.7 with the older version +which was hacked for THINK C version 4. I have conditionalized the code +so that I think it should work with both THINK C version 4 and 5 (for +those of you who don't know: the THINK_C symbol is defined as 1 in version +4 and as 5 in version 5). I have put in some missing prototypes, so it +compiles also with "require prototypes" on. + +Most of the notes below still apply, in particular that about the MakeRes +program. + + +Notes on the THINK C version of Flex +Russell S. Finn, 19-FEB-90 +Internet: rsfinn@athena.mit.edu, rsfinn@neutron.lcs.mit.edu +CompuServe: 76377,1107 +GEnie: RSFINN + +Flex appears to be covered by a copyright notice from the University of +California, similar to the one covering Berkeley Unix; the Free Software +Foundation is not part of the picture here. So here is a version +created with THINK C 4.0, along with the source code; as with the +Bison distribution, I am including *all* of the source code I received +with the package. + +The current version (modification date January 25, 1990) has only the +bare-bones interface provided by the THINK C library routine "ccommand", +which allows the user to type a command line and to redirect the output. +Perhaps someday I may try to implement a "real" user interface; perhaps +not. + +The only modifications made to the source file are surrounded by "#ifdef +THINK_C"..."#endif"; in theory, then, these sources could be recompiled +on another system if necessary. These are the actual files modified: +alloca.c, dfa.c, flexdef.h, main.c, misc.c, scan.c, sym.c. Most of these +changes were minor, and many of them would have been unnecessary if the +original Flex source code had been written for an ANSI-compliant C compiler. +In addition, the file "macutils.c" is completely new; see the discussion +of "MakeRes" below. + +THINK C users may find it convenient to have the output text files written +by Flex be THINK C documents. To do this, create a copy of the "ANSI" +project called "ANSI-KAHL", and a copy of the file "fopen.c" called +"fopen-KAHL.c". In the copy, find the routine "setfiletype", and replace +the lines: + if (!(oflag & F_BINARY)) + pb.ioFlFndrInfo.fdType = 'TEXT'; +with the lines: + if (!(oflag & F_BINARY)) { + pb.ioFlFndrInfo.fdType = 'TEXT'; + pb.ioFlFndrInfo.fdCreator = 'KAHL'; + } +Replace "fopen.c" with the new "fopen-KAHL.c", rebuild the new project +"ANSI-KAHL", and use this project in the project file "Flex.¹" +instead of the "ANSI" project. + +** The "MakeRes" program + +The output files created by Flex contain large amounts of preinitialized +static data; the file "scan.c" contained in the Flex.¹ project is one +such file. However, the Macintosh architecture limits normal applications +to 32K of global data. In many cases (including Flex), this limit can +be surpassed by the static data generated by Flex. + +The solution I have implemented for the THINK C version of Flex is to +extract the data tables from the Flex output file, and paste them into +the file "MakeRes.c". Then, by recompiling and running the program in +the "MakeRes.¹" project (it is not necessary to create an application), +a resource file called "Flex.¹.rsrc" is created in the current directory. +The Flex output file "scan.c" has been modified to load the static data +from the resource fork of the Flex application. This is done by calling +the "load_table" function, which is defined in the file "macutils.c". + +In the application for which I needed Flex, the data tables were small +enough that I didn't need to do this. However, if your application +requires you to do this, simply follow the model of "scan.c"; the MakeRes +project and source code has been included for your use. + diff --git a/MISC/Macintosh/alloca.c b/MISC/Macintosh/alloca.c new file mode 100644 index 0000000..9cb6fa0 --- /dev/null +++ b/MISC/Macintosh/alloca.c @@ -0,0 +1,195 @@ +/* + alloca -- (mostly) portable public-domain implementation -- D A Gwyn + + last edit: 86/05/30 rms + include config.h, since on VMS it renames some symbols. + Use xmalloc instead of malloc. + + This implementation of the PWB library alloca() function, + which is used to allocate space off the run-time stack so + that it is automatically reclaimed upon procedure exit, + was inspired by discussions with J. Q. Johnson of Cornell. + + It should work under any C implementation that uses an + actual procedure stack (as opposed to a linked list of + frames). There are some preprocessor constants that can + be defined when compiling for your specific system, for + improved efficiency; however, the defaults should be okay. + + The general concept of this implementation is to keep + track of all alloca()-allocated blocks, and reclaim any + that are found to be deeper in the stack than the current + invocation. This heuristic does not reclaim storage as + soon as it becomes invalid, but it will do so eventually. + + As a special case, alloca(0) reclaims storage without + allocating any. It is a good idea to use alloca(0) in + your main control loop, etc. to force garbage collection. +*/ +#ifndef lint +static char SCCSid[] = "@(#)alloca.c 1.1"; /* for the "what" utility */ +#endif + +#ifdef emacs +#include "config.h" +#ifdef static +/* actually, only want this if static is defined as "" + -- this is for usg, in which emacs must undefine static + in order to make unexec workable + */ +#ifndef STACK_DIRECTION +you +lose +-- must know STACK_DIRECTION at compile-time +#endif /* STACK_DIRECTION undefined */ +#endif /* static */ +#endif /* emacs */ + +#ifndef alloca /* If compiling with GCC, this file's not needed. */ + +#ifdef __STDC__ +typedef void *pointer; /* generic pointer type */ +#else +typedef char *pointer; /* generic pointer type */ +#endif + +#define NULL 0 /* null pointer constant */ + +extern void free(); +extern pointer xmalloc(); + +/* + Define STACK_DIRECTION if you know the direction of stack + growth for your system; otherwise it will be automatically + deduced at run-time. + + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown +*/ + +#ifndef STACK_DIRECTION +#define STACK_DIRECTION 0 /* direction unknown */ +#endif + +#if STACK_DIRECTION != 0 + +#define STACK_DIR STACK_DIRECTION /* known at compile-time */ + +#else /* STACK_DIRECTION == 0; need run-time code */ + +static int stack_dir; /* 1 or -1 once known */ +#define STACK_DIR stack_dir + +static void +find_stack_direction (/* void */) +{ + static char *addr = NULL; /* address of first + `dummy', once known */ + auto char dummy; /* to get stack address */ + + if (addr == NULL) + { /* initial entry */ + addr = &dummy; + + find_stack_direction (); /* recurse once */ + } + else /* second entry */ + if (&dummy > addr) + stack_dir = 1; /* stack grew upward */ + else + stack_dir = -1; /* stack grew downward */ +} + +#endif /* STACK_DIRECTION == 0 */ + +/* + An "alloca header" is used to: + (a) chain together all alloca()ed blocks; + (b) keep track of stack depth. + + It is very important that sizeof(header) agree with malloc() + alignment chunk size. The following default should work okay. +*/ + +#ifndef ALIGN_SIZE +#define ALIGN_SIZE sizeof(double) +#endif + +typedef union hdr +{ + char align[ALIGN_SIZE]; /* to force sizeof(header) */ + struct + { + union hdr *next; /* for chaining headers */ + char *deep; /* for stack depth measure */ + } h; +} header; + +/* + alloca( size ) returns a pointer to at least `size' bytes of + storage which will be automatically reclaimed upon exit from + the procedure that called alloca(). Originally, this space + was supposed to be taken from the current stack frame of the + caller, but that method cannot be made to work for some + implementations of C, for example under Gould's UTX/32. +*/ + +static header *last_alloca_header = NULL; /* -> last alloca header */ + +pointer +alloca (size) /* returns pointer to storage */ + unsigned size; /* # bytes to allocate */ +{ + auto char probe; /* probes stack depth: */ + register char *depth = &probe; + +#if STACK_DIRECTION == 0 + if (STACK_DIR == 0) /* unknown growth direction */ + find_stack_direction (); +#endif + + /* Reclaim garbage, defined as all alloca()ed storage that + was allocated from deeper in the stack than currently. */ + + { + register header *hp; /* traverses linked list */ + + for (hp = last_alloca_header; hp != NULL;) + if ((STACK_DIR > 0 && hp->h.deep > depth) + || (STACK_DIR < 0 && hp->h.deep < depth)) + { + register header *np = hp->h.next; + + free ((pointer) hp); /* collect garbage */ + + hp = np; /* -> next header */ + } + else + break; /* rest are not deeper */ + + last_alloca_header = hp; /* -> last valid storage */ + } + + if (size == 0) + return NULL; /* no allocation required */ + + /* Allocate combined header + user data storage. */ + + { + register pointer new = xmalloc (sizeof (header) + size); + + /* address of header */ + + ((header *)new)->h.next = last_alloca_header; + ((header *)new)->h.deep = depth; + + last_alloca_header = (header *)new; + + /* User storage begins just after header. */ + + return (pointer)((char *)new + sizeof(header)); + } +} + +#endif /* no alloca */ diff --git a/MISC/Macintosh/alloca.h b/MISC/Macintosh/alloca.h new file mode 100644 index 0000000..f48eaf2 --- /dev/null +++ b/MISC/Macintosh/alloca.h @@ -0,0 +1,10 @@ +/**************** +** alloca.h +** +** header for alloca() +*****************/ + +typedef void *pointer; + +pointer alloca(unsigned size); + diff --git a/MISC/Macintosh/xmalloc.c b/MISC/Macintosh/xmalloc.c new file mode 100644 index 0000000..5bef831 --- /dev/null +++ b/MISC/Macintosh/xmalloc.c @@ -0,0 +1,69 @@ +/* xmalloc.c -- malloc with out of memory checking + Copyright (C) 1990, 1991 Free Software Foundation, Inc. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2, or (at your option) + any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ + +#if STDC_HEADERS || THINK_C +#include <stdlib.h> +#else +char *malloc (); +char *realloc (); +void free (); +#endif + +#ifdef THINK_C +#define error(x, y, z) perror(z) /* Throw out meaningless arguments */ +#else +void error (); +#endif + +/* Allocate N bytes of memory dynamically, with error checking. */ + +char * +xmalloc (n) + unsigned n; +{ + char *p; + + p = malloc (n); + if (p == 0) + /* Must exit with 2 for `cmp'. */ + error (2, 0, "virtual memory exhausted"); + return p; +} + +/* Change the size of an allocated block of memory P to N bytes, + with error checking. + If P is NULL, run xmalloc. + If N is 0, run free and return NULL. */ + +char * +xrealloc (p, n) + char *p; + unsigned n; +{ + if (p == 0) + return xmalloc (n); + if (n == 0) + { + free (p); + return 0; + } + p = realloc (p, n); + if (p == 0) + /* Must exit with 2 for `cmp'. */ + error (2, 0, "virtual memory exhausted"); + return p; +} diff --git a/MISC/NT/Makefile b/MISC/NT/Makefile new file mode 100644 index 0000000..581d497 --- /dev/null +++ b/MISC/NT/Makefile @@ -0,0 +1,103 @@ +# +# make file for "flex" tool +# @(#) $Header: /usr/fsys/odin/a/vern/flex/RCS/Makefile,v 2.9 +# 90/05/26 17:28:44 vern Exp $ (LBL) +# +# the first time around use "nmake f_flex" +# +# This makefile is specific for Microsoft's Visual C 2.0, & nmake +# +# - Stan Adermann <stana@leonardo.lmt.com> +# + + +SKELFLAGS = -DDEFAULT_SKELETON_FILE=\"c:/src/flex/flex.skl\" +CFLAGS = -nologo -W2 -F 8000 -Ox -DUSG +LDFLAGS = /nologo /BATCH /STACK:8000 +FLEX_FLAGS = -ist8 -Sflex.skl + +FLEX = .\flex.exe +CC = cl +YACC = c:\lib\byacc +MAKE = nmake /nologo + +FLEXOBJS = \ + ccl.obj \ + dfa.obj \ + ecs.obj \ + gen.obj \ + main.obj \ + misc.obj \ + nfa.obj \ + parse.obj \ + scan.obj \ + skel.obj \ + sym.obj \ + tblcmp.obj \ + yylex.obj + +FLEX_C_SOURCES = \ + ccl.c \ + dfa.c \ + ecs.c \ + gen.c \ + main.c \ + misc.c \ + nfa.c \ + parse.c \ + scan.c \ + skel.c \ + sym.c \ + tblcmp.c \ + yylex.c + +all : flex.exe + +flex.exe : $(FLEXOBJS) + link $(LDFLAGS) $(FLEXOBJS) -out:$*.exe + +f_flex: + copy initscan.c scan.c + touch scan.c + @echo compiling first flex + $(MAKE) flex.exe + del scan.c + @echo using first flex to generate final version... + $(MAKE) flex.exe + +# +# general inference rule +# +.c.obj: + $(CC) -c $(CFLAGS) $*.c + +parse.h parse.c : parse.y + $(YACC) -d parse.y + @move y_tab.c parse.c + @move y_tab.h parse.h + +scan.c : scan.l + $(FLEX) $(FLEX_FLAGS) $(COMPRESSION) scan.l >scan.c + + +scan.obj : scan.c parse.h flexdef.h + +main.obj : main.c flexdef.h + $(CC) $(CFLAGS) -c $(SKELFLAGS) main.c + +ccl.obj : ccl.c flexdef.h +dfa.obj : dfa.c flexdef.h +ecs.obj : ecs.c flexdef.h +gen.obj : gen.c flexdef.h +misc.obj : misc.c flexdef.h +nfa.obj : nfa.c flexdef.h +parse.obj : parse.c flexdef.h +sym.obj : sym.c flexdef.h +tblcmp.obj : tblcmp.c flexdef.h +yylex.obj : yylex.c flexdef.h +skel.obj : skel.c flexdef.h + + +clean : + del *.obj + del *.map diff --git a/MISC/NT/config.h b/MISC/NT/config.h new file mode 100644 index 0000000..17e831a --- /dev/null +++ b/MISC/NT/config.h @@ -0,0 +1,32 @@ +/* config.h. Generated automatically by configure. */ +/* $Header: /home/daffy/u0/vern/flex/RCS/conf.in,v 1.2 95/01/09 +12:11:51 vern Exp $ */ + +/* Define to empty if the keyword does not work. */ +/* #undef const */ + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +/* #undef size_t */ + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define if you have the <malloc.h> header file. */ +#define HAVE_MALLOC_H 1 + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define if you have <alloca.h> and it should be used (not on +Ultrix). */ +#define HAVE_ALLOCA_H 0 + +/* Define if platform-specific command line handling is necessary. */ +/* #undef NEED_ARGV_FIXUP */ + +/* Define if you use FAT file system, leave undefined for NTFS */ +#undef SHORT_FILE_NAMES +/* #define SHORT_FILE_NAMES 1 */ diff --git a/MISC/NeXT b/MISC/NeXT new file mode 100644 index 0000000..f862a1f --- /dev/null +++ b/MISC/NeXT @@ -0,0 +1,34 @@ +this API is not documented/supported by NeXT and may go away at any time, + so test again when you upgrade (works fine for me on NEXTSTEP_Dev_3.2) +------------------------------------------------------------------------- +See KBNS.32.2.029 from the successor of: +ftp://ftp.cs.orst.edu/software/NeXT/documents/KBNS.32.1.rtf +Otherwise (that successor not yet having been published), ask me for a copy +of the item on RfSchtkt@maze.ruca.ua.ac.be. Bison's messages are not as +disciplined as flex' ones, so it should get more discipline first. + +Specifically (in addition to what's described in the KBNS item): +makeUser.o and make_support.o should be added to OBJECTS in Makefile.in +In parse.y, line_pinpoint() (assumption all messages ultimately go there), add: + make_support( + // don't worry about declaring: cc's source doesn't either, + // it seems + syntaxerror?0:1, + NULL, + infilename, + line, + str, + 0,0,0 + ); + +FMyIO: in cc, these files contain the word make_support: + ChangeLog-NeXT + Makefile.in + config/next.h: defines REPORT_EVENT in terms of make_support + make_support.c + +FMyIO: in cc, these files contain the word REPORT_EVENT: + cccp.c + config/next.h + gcc.c + toplev.c diff --git a/MISC/OS2/Makefile.os2 b/MISC/OS2/Makefile.os2 new file mode 100644 index 0000000..e984f6d --- /dev/null +++ b/MISC/OS2/Makefile.os2 @@ -0,0 +1,72 @@ +# make file for "flex" tool, emx+gcc
+
+release:
+ $(MAKE) -f Makefile.os2 flex.exe \
+ CC="gcc -Zomf -O" O=".obj" A=".lib" AR="emxomfar" \
+ LDFLAGS="-s -Zcrtdll -Zstack 512"
+debug:
+ $(MAKE) -f Makefile.os2 flex.exe \
+ CC="gcc -g" O=".o" A=".a" AR="ar"
+
+CFLAGS = -DOS2 -DSHORT_FILE_NAMES
+
+YACC = bison
+FLEX = flex
+FLEX_FLAGS = -ist
+
+.SUFFIXES: .c $O
+
+.c$O:
+ $(CC) $(CFLAGS) -c $<
+
+FLEXLIB = fl$A
+FLEXOBJS = ccl$O dfa$O ecs$O gen$O main$O misc$O nfa$O parse$O \
+ scan$O skel$O sym$O tblcmp$O yylex$O
+LIBOBJS = libmain$O libyywrap$O
+
+flex.exe : $(FLEXOBJS) $(FLEXLIB)
+ $(CC) $(LDFLAGS) -o $@ $(FLEXOBJS) $(FLEXLIB)
+
+first_flex:
+ cp initscan.c scan.c
+ $(MAKE) $(MFLAGS) flex
+
+$(FLEXLIB): $(LIBOBJS)
+ $(AR) cru $(FLEXLIB) $(LIBOBJS)
+ $(AR) s $(FLEXLIB)
+
+parse.h parse.c: parse.y
+ $(YACC) -d -o parse.c parse.y
+
+scan.c : scan.l
+ $(FLEX) $(FLEX_FLAGS) $(COMPRESSION) scan.l >scan.c
+
+scan$O : scan.c parse.h flexdef.h
+
+main$O : main.c flexdef.h
+ccl$O : ccl.c flexdef.h
+dfa$O : dfa.c flexdef.h
+ecs$O : ecs.c flexdef.h
+gen$O : gen.c flexdef.h
+misc$O : misc.c flexdef.h
+nfa$O : nfa.c flexdef.h
+parse$O : parse.c flexdef.h
+sym$O : sym.c flexdef.h
+tblcmp$O : tblcmp.c flexdef.h
+yylex$O : yylex.c flexdef.h
+
+skel.c: flex.skl mkskel.sh
+ $(SHELL) mkskel.sh flex.skl >skel.c
+
+test : flex
+ flex $(FLEX_FLAGS) $(COMPRESSION) scan.l | diff scan.c -
+
+bigtest :
+ rm -f scan.c ; $(MAKE) COMPRESSION="-C" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-Ce" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-Cm" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-Cfe" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-CFe" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-Cf" test
+ rm -f scan.c ; $(MAKE) COMPRESSION="-CF" test
+ rm -f scan.c ; $(MAKE)
diff --git a/MISC/OS2/config.h b/MISC/OS2/config.h new file mode 100644 index 0000000..acf7b86 --- /dev/null +++ b/MISC/OS2/config.h @@ -0,0 +1,28 @@ +/* ------------------------------------------------ */ +/* version of config.h for OS/2 */ +/* ------------------------------------------------ */ + +/* Define to empty if the keyword does not work. */ +#undef const + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +#undef size_t + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Define if you have the <malloc.h> header file. */ +#define HAVE_MALLOC_H 1 + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define if you have <alloca.h> and it should be used (not on Ultrix). */ +#define HAVE_ALLOCA_H + +/* Define if platform-specific command line handling is necessary. */ +#define NEED_ARGV_FIXUP +#define argv_fixup(ac,av) { _response(ac,av); _wildcard(ac,av);} diff --git a/MISC/README b/MISC/README new file mode 100644 index 0000000..dc16b02 --- /dev/null +++ b/MISC/README @@ -0,0 +1,76 @@ +Miscellaneous flex stuff. The items which have been tested with flex 2.5 are: + + - texinfo/, a subdirectory containing a "texinfo" version of flex(1) + and the corresponding "info" files (contributed by Francois Pinard). + + - VMS/, a subdirectory containing makefiles, configuration files, + run-time support, and installation notes for building flex 2.5 + on VMS (contributed by Pat Rankin). + + - Borland/ - makefile and config.h for Borland 4.02 compiler + (contributed by Terrence O Kane, who notes that no source + code changes were necessary). + + - NT/ - Makefile and config.h for NT, contributed by Stan Adermann. + + - OS2/ - Makefile and config.h for building flex under OS/2, + contributed by Kai Uwe Rommel. + + - Amiga/: notes on building flex for the Amiga, contributed + by Andreas Scherer. + + - parse.c, parse.h - output of running yacc (byacc, actually) + on parse.y. If your system doesn't have a flavor of yacc available, + copy these into the main flex source directory instead. + + - flex.man - preformatted version of flex man page + + +The following have been tested using flex 2.4: + + - debflex.awk, an awk script for anotating flex debug output. + It presently only works with gawk and mawk, not with "old" + or "new" awk. + + - NeXT: ProjectBuilder.app support for use in the NeXT world. + + - Notes on building flex for the Macintosh using Think-C, + in the Macintosh/ subdirectory. + + - testxxLexer.l, a sample C++ program that uses flex's scanner + class option ("-+"). + + - fastwc/, a subdirectory containing examples of how to use flex + to write progressively higher-performance versions of the Unix + "wc" utility. This certainly should work with 2.5, but hasn't + been tested. + + - Borland.old/: notes on building flex 2.4 for Borland C++ 3.1 + on MS-DOS. These shouldn't be needed for flex 2.5. Included + only in case you encounter unanticipated difficulties. + + - EBCDIC: contact information for building flex for EBCDIC. + + +The following are all out-of-date with respect to flex release 2.4 (and +in general up-to-date for flex 2.3): + + - Atari/Atari.patches, patches for porting flex to the Atari and + to Minix. + + - A number of notes and Makefiles for compiling flex under MS-DOS, + in the MSDOS/ subdirectory. + + - Notes on building flex for MVS, in the MVS/ subdirectory. + +If any of this is out-of-date and can be deleted, please let me know. + +And the following is included for compatibility with some broken versions +of bison: + + - alloca.c, a public-domain, mostly-portable version of the + alloca() routine (used by bison's parsers) written by D. A. Gwyn. + + +Many thanks to those who contributed these files. Updated versions will +be appreciated! diff --git a/MISC/VMS/README.VMS b/MISC/VMS/README.VMS new file mode 100644 index 0000000..e4a4966 --- /dev/null +++ b/MISC/VMS/README.VMS @@ -0,0 +1,83 @@ +Brief instructions for building flex 2.5.x for VMS: + + 0) if you have either MMS (from Digital) or MMK (freeware) for use +as a `make' utility, follow the directions in steps #1 through #5 below. +If not, execute + @BUILD.COM xxxC +where "xxxC" is either "VAXC" or "DECC" or "GNUC", and then skip to +step #5. + + 1) set default to the source directory (not the [.MISC.VMS] subdirectory +where this file is located). + + 2) COPY [.MISC.VMS]DESCRIP.MMS []*.* +(Recursive invocations of `make' for the `bigcheck' test assume that the +makefile will be found as descrip.mms in the current directory.) + +To build with VAX C for VAX/VMS: + 3) MMS /MACRO=("VAXC=1") FLEX.EXE +(The /macro qualifier is optional in this case.) + +To build with GNU C for VAX/VMS: + 2.5) possibly edit descrip.mms to uncomment `SET COMMAND' for GCCINIT, + depending on local site configuration + 3) MMS /MACRO=("GNUC=1") FLEX.EXE + +To build with DEC C for either VAX/VMS or Alpha/VMS: + 3) MMS /MACRO=("DECC=1") FLEX.EXE +(Expect one or two informational messages from the compiler about +implicitly declared functions.) + +Minimal testing of the resulting program: + 4) MMS CHECK +(If `diff' reports no warnings, the test has succeeded.) + +More thorough testing: + 4.5) MMS /MACRO=("xxxC=1") BIGCHECK ! "xxxC=1" as in step #3 above +(If using an older version of MMK rather than MMS, this might fail when +`make' is invoked recursively due to excessive BYTLM usage by MMK.) + +Installation (the VMS makefile does not support an `install' target; +you'll need to do this part manually): + 5) copy flex.exe, flex.doc, flex.skl, flexlib.olb, and FlexLexer.h to +location(s) appropriate for your site. To use flex, define a "foreign" +command by making a DCL symbol whose value begins with a dollar sign +immediately followed by the filename for flex.exe, as in + $ flex :== $local_tools:flex.exe +where `local_tools:' is the logical name pointing to flex.exe's location. +This symbol will ordinarily be a candidate for your login.com. When +invoking flex, upper- or mixed-case command line options must be enclosed +in quotes. For example, + $ flex "-Pxyz" "-L" -t mylexer.l > mylexer.c +(use prefix "xyz" instead of "yy", suppress `#line' compiler directives +in the output, write the output to `stdout', process file mylexer.l, +and capture `stdout' in file mylexer.c). As illustrated here, this VMS +version of flex supports emulation of command line I/O redirection used +by Unix shells. + + flex.exe -- the executable image for the flex program; + flex.doc -- documentation, the "man page" describing flex (flex.1 + processed with `nroff -man' followed by `col -b'); + flex.skl -- a text file containing flex's default skeleton; + with this version of flex, it is for reference only; + flex.exe does not need to know where to find it; + flexlib.olb -- an object library containing some support routines; + you might need to link your generated lexer against + it, depending on how your program is designed; + flex.exe does not access it; it corresponds to + `libfl.a' under Unix; + FlexLexer.h -- header file used for C++ class-based lexers; not + needed for ordinary C lexers. + +Notes: + This VMS port of flex supports only the original Unix command line +interface, not the native DCL interface which was available for flex 2.3. + + build.com -- DCL command procedure as alternative to descrip.mms; + descrip.mms -- 2.5.x makefile for use with MMS or MMK (see step #1); + mkskel.tpu -- TPU program used to make skel.c from flex.skl for full + build from scratch; performs same function as mkskel.sh; + vms-conf.h -- pre-configured `conf.in', copied to [-.-]config.h; + vms-code.c -- VMS-specific support code, copied to [-.-]vms-code.c; + README.VMS -- this file + diff --git a/MISC/VMS/build.com b/MISC/VMS/build.com new file mode 100644 index 0000000..dbde55a --- /dev/null +++ b/MISC/VMS/build.com @@ -0,0 +1,155 @@ +$! VMS build procedure for flex 2.5.x; +$ v = 'f$verify(0)' +$! +$! usage: +$! $ @[.MISC.VMS]BUILD.COM compiler parser [test] +$! where `compiler' is either "GNUC" or "DECC" or "VAXC" or empty +$! and `parser' is either "BISON" or "BYACC" or "YACC" or empty +$! and `[test]' is either "CHECK-ONLY" or "NO-CHECK" or empty +$! empty compiler defaults to VAX C (even under Alpha/VMS); +$! special "LINK" compiler value does link without compilation; +$! empty parser defaults to using supplied parse code in [.MISC]; +$! optional test is performed by default. +$! +$ +$! we start from [.MISC.VMS], then move to the main source directory +$ where = f$parse("_._;",f$environ("PROCEDURE")) - "_._;" +$ set default 'where' +$ brkt = f$extract(f$length(where)-1,1,where) +$ if f$locate(".MISC.VMS"+brkt,where).lt.f$length(where) then - + set default 'f$string(f$extract(0,1,f$dir()) + "-.-" + brkt)' +$ +$ p1 := 'p1' +$ p2 := 'p2' +$ p3 := 'p3' +$ if p1.eqs."LINK" then goto link +$ if p3.eqs."CHECK-ONLY" then goto check +$ p2 = p2 - "_PARSER" +$! +$ CDEFS = "/Define=(""VMS"")" ! =(""VMS"",""DEFAULT_CSIZE=256"") +$! +$ if p1.eqs."GNUC" +$ then CC = "gcc" +$ CFLAGS = "/noList/Opt=2/Debug/noVerbose" +$ LIBS = "gnu_cc:[000000]gcclib.olb/Library, sys$library:vaxcrtl.olb/Library" +$ else CC = "cc" +$ if p1.eqs."DECC" +$ then CFLAGS = "/noList/Prefix=All" +$ LIBS = "" +$ if f$trnlnm("DECC$CC_DEFAULT").nes."" then CC = CC + "/DECC" +$ else CFLAGS = "/noList/Optimize=noInline" +$ LIBS = "sys$share:vaxcrtl.exe/Shareable" +$ if f$trnlnm("DECC$CC_DEFAULT").nes."" then CC = CC + "/VAXC" +$ if p1.nes."" .and. p1.nes."VAXC" then exit %x002C +$ endif +$ endif +$! +$ no_parser = 0 +$ if p2.eqs."BISON" +$ then YACC = "bison" +$ YACCFLAGS = "/Defines/Fixed_Outfiles" +$ ALLOCA = ",[]alloca.obj" +$ else +$ YACCFLAGS = "-d" +$ ALLOCA = "" +$ if p2.eqs."BYACC" .or. p2.eqs."YACC" +$ then YACC = f$edit(p2,"LOWERCASE") +$ else YACC = "! yacc" +$ if p2.nes."" .and. p2.nes."NO" .and. p2.nes."NONE" then exit %x002C +$ no_parser = 1 +$ endif +$ endif +$! +$ ECHO = "write sys$output" +$ COPY = "copy_" +$ MOVE = "rename_/New_Vers" +$ MUNG = "search_/Exact/Match=NOR" +$ PURGE = "purge_/noConfirm/noLog" +$ REMOVE = "delete_/noConfirm/noLog" +$ TPU = "edit_/TPU/noJournal/noDisplay/noSection" +$! +$ if v then set verify +$! +$ 'COPY' [.misc.vms]vms-conf.h config.h +$ 'COPY' [.misc.vms]vms-code.c vms-code.c +$ 'COPY' [.misc]flex.man flex.doc +$ if ALLOCA.nes."" then 'COPY' [.MISC]alloca.c alloca.c +$ 'COPY' initscan.c scan.c !make.bootstrap +$! +$ if f$search("skel.c").nes."" then - + if f$cvtime(f$file_attr("skel.c","RDT")).gts. - + f$cvtime(f$file_attr("flex.skl","RDT")) then goto skip_mkskel +$ 'TPU' /Command=[.misc.vms]mkskel.tpu flex.skl /Output=skel.c +$skip_mkskel: +$! +$ if f$search("parse.c").nes."" .and. f$search("parse.h").nes."" then - + if f$cvtime(f$file_attr("parse.c","RDT")).gts. - + f$cvtime(f$file_attr("parse.y","RDT")) then goto skip_yacc +$ if f$search("y_tab.%").nes."" then 'REMOVE' y_tab.%;* +$ if no_parser +$ then 'COPY' [.misc]parse.% sys$disk:[]y_tab.* +$ else 'YACC' 'YACCFLAGS' parse.y +$ endif +$ 'MUNG' y_tab.c "#module","#line" /Output=parse.c +$ 'REMOVE' y_tab.c;* +$ 'MOVE' y_tab.h parse.h +$skip_yacc: +$! +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] ccl.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] dfa.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] ecs.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] gen.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] main.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] misc.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] nfa.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] parse.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] scan.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] skel.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] sym.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] tblcmp.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] yylex.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] vms-code.c +$ if ALLOCA.nes."" then - !bison + 'CC' 'CFLAGS' /Define=("STACK_DIRECTION=-1","xmalloc=yy_flex_xmalloc") alloca.c +$! +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] libmain.c +$ 'CC' 'CFLAGS' 'CDEFS' /Include=[] libyywrap.c +$ library/Obj flexlib.olb/Create libmain.obj,libyywrap.obj/Insert +$ if f$search("flexlib.olb;-1").nes."" then 'PURGE' flexlib.olb +$! +$ open/Write optfile sys$disk:[]crtl.opt +$ write optfile LIBS +$ close optfile +$ if f$search("crtl.opt;-1").nes."" then 'PURGE' crtl.opt +$! +$ version = "# flex ""2.5""" !default, overridden by version.h +$ open/Read/Error=v_h_2 hfile version.h +$ read/End=v_h_1 hfile version +$v_h_1: close/noLog hfile +$v_h_2: version = f$element(1,"""",version) +$ open/Write optfile sys$disk:[]ident.opt +$ write optfile "identification=""flex ''version'""" +$ close optfile +$ if f$search("ident.opt;-1").nes."" then 'PURGE' ident.opt +$! +$link: +$ link/noMap/Exe=flex.exe ccl.obj,dfa.obj,ecs.obj,gen.obj,main.obj,misc.obj,- + nfa.obj,parse.obj,scan.obj,skel.obj,sym.obj,tblcmp.obj,yylex.obj,- + vms-code.obj 'ALLOCA' ,flexlib.olb/Lib,- + sys$disk:[]crtl.opt/Opt,sys$disk:[]ident.opt/Opt +$! +$ if p3.eqs."NO-CHECK" .or. p3.eqs."NOCHECK" then goto done +$ +$check: +$ 'ECHO' "" +$ 'ECHO' " Checking with COMPRESSION=""""" +$ mcr sys$disk:[]flex.exe -t -p scan.l > scan.chk +$ diff_/Output=_NL:/Maximum_Diff=1 scan.c scan.chk +$ if $status +$ then 'ECHO' " Test passed." +$ 'REMOVE' scan.chk;* +$ else 'ECHO' "? Test failed!" +$ endif +$ +$done: +$ exit diff --git a/MISC/VMS/descrip.mms b/MISC/VMS/descrip.mms new file mode 100644 index 0000000..5b2859d --- /dev/null +++ b/MISC/VMS/descrip.mms @@ -0,0 +1,311 @@ +# descrip.mms -- makefile for building `flex' using MMS or MMK on VMS; +# created manually from Makefile.in +# flex 2.5.0 Jan'95 + +MAKEFILE = descrip.mms # from [.MISC.VMS] +MAKE = $(MMS) /Descr=$(MAKEFILE) +MAKEFLAGS = $(MMSQUALIFIERS) + +# Possible values for DEFS: +# "VMS" -- used just to make sure parentheses aren't empty; +# For flex to always generate 8-bit scanners, append +# ,"DEFAULT_CSIZE=256" inside /Define=() of DEFS. + +DEFS = /Define=("VMS") +LDFLAGS = /noMap + +# compiler handling +.ifdef GNUC +CC = gcc +GCCINIT = ! SET COMMAND GNU_CC:[000000]GCC +CFLAGS = /noList/Opt=2/Debug/noVerbose +LIBS = gnu_cc:[000000]gcclib.olb/Library, sys$library:vaxcrtl.olb/Library +C_CHOICE = "GNUC=1" +.else ! not GNU C +CC = cc +GCCINIT = +.ifdef DECC +CFLAGS = /noList/Prefix=All +LIBS = +C_CHOICE = "DECC=1" +.else ! not DEC C; assume VAX C +CFLAGS = /noList/Optimize=noInline +LIBS = sys$share:vaxcrtl.exe/Shareable +C_CHOICE = "VAXC=1" +.endif +.endif + +# parser handling +# mms/macro=("xxxC=1","zzz_parser=1"), where "zzz_parser" is +# either "bison_parser" or "byacc_parser" or "yacc_parser", +# otherwise assumed to be "no_parser"; and where "xxxC=1" is +# either "VAXC=1", "GNUC=1", or "DECC=1" as above +.ifdef bison_parser +YACC = bison +YACCFLAGS = /Defines/Fixed_Outfiles +YACCINIT = set command gnu_bison:[000000]bison +ALLOCA = ,[]alloca.obj # note leading comma +.else +YACCFLAGS = -d +YACCINIT = +ALLOCA = +.ifdef byacc_parser +YACC = byacc +.else +.ifdef yacc_parser +YACC = yacc +.else +# none of bison, byacc, or yacc specified +.ifdef no_parser +.else +no_parser=1 +.endif #<none> +.endif #yacc +.endif #byacc +.endif #bison + +# VMS-specific hackery +ECHO = write sys$output # requires single quoted arg +COPY = copy_ # +MOVE = rename_/New_Vers # within same device only +MUNG = search_/Exact/Match=NOR # to strip unwanted `#module' directive +NOOP = continue # non-empty command that does nothing +PURGE = purge_/noConfirm/noLog # relatively quiet file removal +REMOVE = delete_/noConfirm/noLog # ditto +TOUCH = append_/New _NL: # requires single file arg +TPU = edit_/TPU/noJournal/noDisplay/noSection + +# You can define this to be "lex.exe" if you want to replace lex at your site. +FLEX =flex.exe +# note: there should be no whitespace between `=' and the name, +# or else $(FLEX_EXEC) below will not function properly. +FLEXLIB = flexlib.olb + +# You normally do not need to modify anything below this point. +# ------------------------------------------------------------ + +VMSDIR = [.MISC.VMS] +MISCDIR = [.MISC] +CURDIR = sys$disk:[] + +CPPFLAGS = $(DEFS)/Include=[] +LIBOPT = $(CURDIR)crtl.opt # run-time library(s) +ID_OPT = $(CURDIR)ident.opt # version identification + +.SUFFIXES : # avoid overhead of umpteen built-in rules +.SUFFIXES : .obj .c + +.c.obj : + $(CC)$(CFLAGS)$(CPPFLAGS) $< + +VMSHDRS = $(VMSDIR)vms-conf.h # copied to []config.h +VMSSRCS = $(VMSDIR)vms-code.c # copied to []vms-code.c +VMSOBJS = ,vms-code.obj # note leading comma + +HEADERS = flexdef.h version.h + +SOURCES = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.y \ + scan.l skel.c sym.c tblcmp.c yylex.c +OBJECTS = ccl.obj,dfa.obj,ecs.obj,gen.obj,main.obj,misc.obj,nfa.obj,parse.obj,\ + scan.obj,skel.obj,sym.obj,tblcmp.obj,yylex.obj $(VMSOBJS) $(ALLOCA) + +LIBSRCS = libmain.c libyywrap.c +LIBOBJS = libmain.obj,libyywrap.obj + +LINTSRCS = ccl.c dfa.c ecs.c gen.c main.c misc.c nfa.c parse.c \ + scan.c skel.c sym.c tblcmp.c yylex.c + +DISTFILES = README NEWS COPYING INSTALL FlexLexer.h \ + configure.in conf.in Makefile.in mkskel.sh flex.skl \ + $(HEADERS) $(SOURCES) $(LIBSRCS) MISC \ + flex.1 scan.c install.sh mkinstalldirs configure + +DIST_NAME = flex + +# flex options to use when generating scan.c from scan.l +COMPRESSION = +PERF_REPORT = -p +# which "flex" to use to generate scan.c from scan.l +FLEX_EXEC = mcr $(CURDIR)$(FLEX) +FLEX_FLAGS = -t $(PERF_REPORT) #$(COMPRESSION) + +MARKER = make.bootstrap + +##### targets start here ##### + +all : $(FLEX) flex.doc + @ $(NOOP) + +install : $(FLEX) flex.doc flex.skl $(FLEXLIB) FlexLexer.h + @ $(ECHO) "-- Installation must be done manually." + @ $(ECHO) " $+" + +.ifdef GCCINIT +.FIRST + $(GCCINIT) + +.endif #GCCINIT + +flex : $(FLEX) + @ $(NOOP) + +$(FLEX) : $(MARKER) $(OBJECTS) $(FLEXLIB) $(LIBOPT) $(ID_OPT) + $(LINK)/Exe=$(FLEX) $(LDFLAGS)\ + $(OBJECTS),$(FLEXLIB)/Lib,$(LIBOPT)/Opt,$(ID_OPT)/Opt + +$(MARKER) : initscan.c + @- if f$search("scan.c").nes."" then $(REMOVE) scan.c;* + $(COPY) initscan.c scan.c + @ $(TOUCH) $(MARKER) + +parse.c : parse.y + @- if f$search("y_tab.%").nes."" then $(REMOVE) y_tab.%;* +.ifdef no_parser + $(COPY) $(MISCDIR)parse.% $(CURDIR)y_tab.* +.else + $(YACCINIT) + $(YACC) $(YACCFLAGS) parse.y +.endif + $(MUNG) y_tab.c "#module","#line" /Output=parse.c + @- $(REMOVE) y_tab.c;* + $(MOVE) y_tab.h parse.h + +parse.h : parse.c + @ $(TOUCH) parse.h + +scan.c : scan.l + $(FLEX_EXEC) $(FLEX_FLAGS) $(COMPRESSION) scan.l > scan.c + +scan.obj : scan.c parse.h flexdef.h config.h +yylex.obj : yylex.c parse.h flexdef.h config.h + +skel.c : flex.skl $(VMSDIR)mkskel.tpu + $(TPU) /Command=$(VMSDIR)mkskel.tpu flex.skl /Output=skel.c + +main.obj : main.c flexdef.h config.h version.h +ccl.obj : ccl.c flexdef.h config.h +dfa.obj : dfa.c flexdef.h config.h +ecs.obj : ecs.c flexdef.h config.h +gen.obj : gen.c flexdef.h config.h +misc.obj : misc.c flexdef.h config.h +nfa.obj : nfa.c flexdef.h config.h +parse.obj : parse.c flexdef.h config.h +skel.obj : skel.c flexdef.h config.h +sym.obj : sym.c flexdef.h config.h +tblcmp.obj : tblcmp.c flexdef.h config.h +vms-code.obj : vms-code.c flexdef.h config.h + +[]alloca.obj : alloca.c + $(CC)$(CFLAGS)/Define=("STACK_DIRECTION=-1","xmalloc=yy_flex_xmalloc") alloca.c + +alloca.c : $(MISCDIR)alloca.c + $(COPY) $(MISCDIR)alloca.c alloca.c + +config.h : $(VMSDIR)vms-conf.h + $(COPY) $(VMSDIR)vms-conf.h config.h + +vms-code.c : $(VMSDIR)vms-code.c + $(COPY) $(VMSDIR)vms-code.c vms-code.c + +test : check + @ $(NOOP) +check : $(FLEX) + @ $(ECHO) "" + @ $(ECHO) " Checking with COMPRESSION="$(COMPRESSION)"" + $(FLEX_EXEC) $(FLEX_FLAGS) $(COMPRESSION) scan.l > scan.chk + diff_/Output=_NL:/Maximum_Diff=1 scan.c scan.chk + +bigcheck : + @- if f$search("scan.c").nes."" then $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-C""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-Ce""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-Cm""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-f""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-Cfea""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-CFer""") check + @- $(REMOVE) scan.c;* + $(MAKE)$(MAKEFLAGS) /Macro=($(C_CHOICE),"COMPRESSION=""-l""","PERF_REPORT=") check + @- $(REMOVE) scan.c;*,scan.chk;* + $(MAKE)$(MAKEFLAGS) $(FLEX) + @- $(PURGE) scan.obj + @ $(ECHO) "All checks successful" + +$(FLEXLIB) : $(LIBOBJS) + library/Obj $(FLEXLIB)/Create $(LIBOBJS)/Insert + @ if f$search("$(FLEXLIB);-1").nes."" then $(PURGE) $(FLEXLIB) + +# We call it .doc instead of .man, to lessen culture shock. :-} +# If MISC/flex.man is out of date relative to flex.1, there's +# not much we can do about it with the tools readily available. +flex.doc : flex.1 + @ if f$search("$(MISCDIR)flex.man").eqs."" then \ + $(COPY) flex.1 $(MISCDIR)flex.man + $(COPY) $(MISCDIR)flex.man flex.doc + +# +# This is completely VMS-specific... +# + +# Linker options file specifying run-time library(s) to link against; +# choice depends on which C compiler is used, and might be empty. +$(LIBOPT) : $(MAKEFILE) + @ open/Write optfile $(LIBOPT) + @ write optfile "$(LIBS)" + @ close optfile + +# Linker options file putting the version number where the ANALYZE/IMAGE +# command will be able to find and report it; assumes that the first line +# of version.h has the version number enclosed within the first and second +# double quotes on it [as in ``#define FLEX_VERSION "2.5.0"'']. +$(ID_OPT) : version.h + @ version = "# flex ""2.5""" !default, overridden by version.h + @- open/Read hfile version.h + @- read hfile version + @- close/noLog hfile + @ version = f$element(1,"""",version) + @ open/Write optfile $(ID_OPT) + @ write optfile "identification=""flex ''version'""" + @ close optfile + + +# +# This is the only stuff moderately useful from the remainder +# of Makefile.in... +# + +mostlyclean : + @- if f$search("scan.chk").nes."" then $(REMOVE) scan.chk;* + @- if f$search("*.obj;-1").nes."" then $(PURGE) *.obj + @- if f$search("*.exe;-1").nes."" then $(PURGE) *.exe + @- if f$search("*.opt;-1").nes."" then $(PURGE) *.opt + +clean : mostlyclean + @- if f$search("*.obj").nes."" then $(REMOVE) *.obj;* + @- if f$search("parse.h").nes."" then $(REMOVE) parse.h;* + @- if f$search("parse.c").nes."" then $(REMOVE) parse.c;* + @- if f$search("alloca.c").nes."" .and.- + f$search("$(MISCDIR)alloca.c").nes."" then $(REMOVE) alloca.c;* + @- if f$search("$(LIBOPT)").nes."" then $(REMOVE) $(LIBOPT);* + @- if f$search("$(ID_OPT)").nes."" then $(REMOVE) $(ID_OPT);* + +distclean : clean + @- if f$search("$(MARKER)").nes."" then $(REMOVE) $(MARKER);* + @- if f$search("$(FLEX)").nes."" then $(REMOVE) $(FLEX);* + @- if f$search("$(FLEXLIB)").nes."" then $(REMOVE) $(FLEXLIB);* + @- if f$search("flex.doc").nes."" then $(REMOVE) flex.doc;* + @- if f$search("scan.c").nes."" then $(REMOVE) scan.c;* + @- if f$search("vms-code.c").nes."" .and.- + f$search("$(VMSDIR)vms-code.c").nes."" then $(REMOVE) vms-code.c;* + @- if f$search("config.h").nes."" .and.- + f$search("$(VMSDIR)vms-conf.h").nes."" then $(REMOVE) config.h;* +# @- if f$search("descrip.mms").nes."" .and.- +# f$search("$(VMSDIR)descrip.mms").nes."" then $(REMOVE) descrip.mms;* + +realclean : distclean + @- if f$search("skel.c").nes."" then $(REMOVE) skel.c;* + diff --git a/MISC/VMS/mkskel.tpu b/MISC/VMS/mkskel.tpu new file mode 100644 index 0000000..e20a0eb --- /dev/null +++ b/MISC/VMS/mkskel.tpu @@ -0,0 +1,45 @@ +! mkskel.tpu +! usage: +! edit/TPU/noDisplay/noSection/Command=mkskel.tpu flex.skl /Output=skel.c +! +! Create a C source file from the flex skeleton data. Copy the file, +! changing backslash (\) to doubled backslash (\\) and quote (") +! to backslash quote (\"). For each line, insert space+space+quote +! at the beginning and quote+comma at the end. Bracket the updated +! text with several lines of prologue and epilogue. +! + skelfile := CREATE_BUFFER("file", GET_INFO(COMMAND_LINE, "file_name")); + SET(NO_WRITE, skelfile); + target := '"' | '\'; !do this once, outside loops + POSITION(BEGINNING_OF(skelfile)); !start here + rest_of_line := CREATE_RANGE(MARK(NONE), MARK(NONE)); !also outside loops + LOOP + EXITIF MARK(NONE) = END_OF(skelfile); !are we done yet? + COPY_TEXT(' "'); start_pos := MARK(NONE); + POSITION(LINE_END); end_pos := MARK(NONE); + MODIFY_RANGE(rest_of_line, start_pos, end_pos); + LOOP + next_match := SEARCH_QUIETLY(target, FORWARD, EXACT, rest_of_line); + EXITIF next_match = 0; + POSITION(BEGINNING_OF(next_match)); + COPY_TEXT('\'); MOVE_HORIZONTAL(1); !past the matched character + MODIFY_RANGE(rest_of_line, MARK(NONE), end_pos); + ENDLOOP; + POSITION(LINE_END); COPY_TEXT('",'); + MOVE_VERTICAL(1); POSITION(LINE_BEGIN); !go to next line + ENDLOOP; + + POSITION(BEGINNING_OF(skelfile)); !insert five line prologue + COPY_TEXT('/* File created from flex.skl via mkskel.tpu */'); SPLIT_LINE; + SPLIT_LINE; + COPY_TEXT('#include "flexdef.h"'); SPLIT_LINE; + SPLIT_LINE; + COPY_TEXT('const char *skel[] = {'); SPLIT_LINE; + + POSITION(END_OF(skelfile)); !append two line epilogue + COPY_TEXT(' 0'); SPLIT_LINE; + COPY_TEXT('};'); !! SPLIT_LINE; + + WRITE_FILE(skelfile, GET_INFO(COMMAND_LINE, "output_file")); + QUIT +!--<eof>-- diff --git a/MISC/VMS/vms-code.c b/MISC/VMS/vms-code.c new file mode 100644 index 0000000..825a6b3 --- /dev/null +++ b/MISC/VMS/vms-code.c @@ -0,0 +1,152 @@ +/* vms-code.c -- additional VMS-specific support code for flex + */ + +#include "flexdef.h" + +static const char *original_arg0; +static const char default_arg0[] = "flex.exe"; + +#define IN_FD 0 +#define OUT_FD 1 +#define ERR_FD 2 + +static char *fix_arg0 PROTO((const char *)); + +/* Command line arguments fixup -- simplify argv[0], and handle `>' + output redirection request; called first thing from main(). */ + +void argv_fixup( iargc, iargv ) +int *iargc; +char ***iargv; +{ + const char *mode[3], *rfm[3], *name[3]; + char *p; + int i, oargc, punct, which, append, alt_rfm; + + /* + * Get original argv[0] supplied by run-time library startup code, + * then replace it with a stripped down one. + */ + original_arg0 = (*iargv)[0]; + (*iargv)[0] = fix_arg0(original_arg0); + + /* + * Check command line arguments for redirection request(s). + * For simplicity, if multiple attempts are made, the last one wins. + */ + name[0] = name[1] = name[2] = 0; + oargc = 1; /* number of args caller will see; count includes argv[0] */ + for (i = 1; i < *iargc; i++) { + p = (*iargv)[i]; + switch (*p) { + case '<': + /* might be "<dir>file"; then again, perhaps "<<dir>file" */ + punct = (strchr(p, '>') != 0); + if (p[1] == '<') { + if (!punct || p[2] == '<') + flexerror("<<'sentinel' input not supported."); + punct = 0; + } + if (punct) /* the '<' seems to be directory punctuation */ + goto arg; /*GOTO*/ + mode[IN_FD] = "r"; + rfm[IN_FD] = 0; + name[IN_FD] = ++p; + if (!*p && (i + 1) < *iargc) + name[IN_FD] = (*iargv)[++i]; + break; + case '>': + append = (p[1] == '>'); + if (append) ++p; + alt_rfm = (p[1] == '$'); + if (alt_rfm) ++p; + which = (p[1] == '&' ? ERR_FD : OUT_FD); + if (which == ERR_FD) ++p; + mode[which] = append ? "a" : "w"; + rfm[which] = alt_rfm ? "rfm=var" : "rfm=stmlf"; + name[which] = ++p; + if (!*p && (i + 1) < *iargc) + name[which] = (*iargv)[++i]; + break; + case '|': + flexerror("pipe output not supported."); + /*NOTREACHED*/ + break; + default: + arg: /* ordinary option or argument */ + (*iargv)[oargc++] = p; + break; + } + } + /* perform any requested redirection; don't bother with SYS$xxx logicals */ + if (name[IN_FD]) + if (!freopen(name[IN_FD], mode[IN_FD], stdin)) + lerrsf("failed to redirect `stdin' from \"%s\"", name[IN_FD]); + if (name[OUT_FD]) + if (!freopen(name[OUT_FD], mode[OUT_FD], stdout, + rfm[OUT_FD], "rat=cr", "mbc=32", "shr=nil")) + lerrsf("failed to redirect `stdout' to \"%s\"", name[OUT_FD]); + if (name[ERR_FD]) /* likely won't see message if this fails; oh well... */ + if (!freopen(name[ERR_FD], mode[ERR_FD], stderr, + rfm[ERR_FD], "rat=cr")) + lerrsf("failed to redirect `stderr' to \"%s\"", name[ERR_FD]); + /* remove any excess arguments (used up from redirection) */ + while (*iargc > oargc) + (*iargv)[--*iargc] = 0; + /* all done */ + return; +} + +/* Pick out the basename of a full filename, and return a pointer + to a modifiable copy of it. */ + +static char *fix_arg0( arg0 ) +const char *arg0; +{ + char *p, *new_arg0; + + if (arg0) { + /* strip off the path */ + if ((p = strrchr(arg0, ':')) != 0) /* device punctuation */ + arg0 = p + 1; + if ((p = strrchr(arg0, ']')) != 0) /* directory punctuation */ + arg0 = p + 1; + if ((p = strrchr(arg0, '>')) != 0) /* alternate dir punct */ + arg0 = p + 1; + } + if (!arg0 || !*arg0) + arg0 = default_arg0; + /* should now have "something.exe;#"; make a modifiable copy */ + new_arg0 = copy_string(arg0); + + /* strip off ".exe" and/or ";#" (version number), + unless it ended up as the whole name */ + if ((p = strchr(new_arg0, '.')) != 0 && (p > new_arg0) + && (p[1] == 'e' || p[1] == 'E') + && (p[2] == 'x' || p[2] == 'X') + && (p[3] == 'e' || p[3] == 'E') + && (p[4] == ';' || p[4] == '.' || p[4] == '\0')) + *p = '\0'; + else if ((p = strchr(new_arg0, ';')) != 0 && (p > new_arg0)) + *p = '\0'; + + return new_arg0; +} + + +#include <ssdef.h> +#include <stsdef.h> + +#ifdef exit +#undef exit +extern void exit PROTO((int)); /* <stdlib.h> ended up prototyping vms_exit */ +#endif + +/* Convert zero to VMS success and non-zero to VMS failure. The latter + does not bother trying to distinguish between various failure reasons. */ + +void vms_exit( status ) +int status; +{ + exit( status == 0 ? SS$_NORMAL : (SS$_ABORT | STS$M_INHIB_MSG) ); +} diff --git a/MISC/VMS/vms-conf.h b/MISC/VMS/vms-conf.h new file mode 100644 index 0000000..65aa477 --- /dev/null +++ b/MISC/VMS/vms-conf.h @@ -0,0 +1,32 @@ +/* config.h manually constructed for VMS */ + +/* Define to empty if the keyword does not work. */ +#undef const + +/* Define to `unsigned' if <sys/types.h> doesn't define. */ +#undef size_t + +/* Define if you have the ANSI C header files. */ +#define STDC_HEADERS + +/* Define if you have the <malloc.h> header file. */ +#undef HAVE_MALLOC_H + +/* Define if you have the <string.h> header file. */ +#define HAVE_STRING_H + +/* Define if you have the <sys/types.h> header file. */ +#ifndef __GNUC__ +#undef HAVE_SYS_TYPES_H +#else +#define HAVE_SYS_TYPES_H +#endif + +/* Define if you have <alloca.h> and it should be used (not on Ultrix). */ +#undef HAVE_ALLOCA_H + +/* Extra platform-specific command line handling. */ +#define NEED_ARGV_FIXUP + +/* Override default exit behavior. */ +#define exit vms_exit diff --git a/MISC/alloca.c b/MISC/alloca.c new file mode 100644 index 0000000..bd4932a --- /dev/null +++ b/MISC/alloca.c @@ -0,0 +1,484 @@ +/* alloca.c -- allocate automatically reclaimed memory + (Mostly) portable public-domain implementation -- D A Gwyn + + This implementation of the PWB library alloca function, + which is used to allocate space off the run-time stack so + that it is automatically reclaimed upon procedure exit, + was inspired by discussions with J. Q. Johnson of Cornell. + J.Otto Tennant <jot@cray.com> contributed the Cray support. + + There are some preprocessor constants that can + be defined when compiling for your specific system, for + improved efficiency; however, the defaults should be okay. + + The general concept of this implementation is to keep + track of all alloca-allocated blocks, and reclaim any + that are found to be deeper in the stack than the current + invocation. This heuristic does not reclaim storage as + soon as it becomes invalid, but it will do so eventually. + + As a special case, alloca(0) reclaims storage without + allocating any. It is a good idea to use alloca(0) in + your main control loop, etc. to force garbage collection. */ + +#ifdef HAVE_CONFIG_H +#if defined (emacs) || defined (CONFIG_BROKETS) +#include <config.h> +#else +#include "config.h" +#endif +#endif + +/* If compiling with GCC 2, this file's not needed. */ +#if !defined (__GNUC__) || __GNUC__ < 2 + +/* If someone has defined alloca as a macro, + there must be some other way alloca is supposed to work. */ +#ifndef alloca + +#ifdef emacs +#ifdef static +/* actually, only want this if static is defined as "" + -- this is for usg, in which emacs must undefine static + in order to make unexec workable + */ +#ifndef STACK_DIRECTION +you +lose +-- must know STACK_DIRECTION at compile-time +#endif /* STACK_DIRECTION undefined */ +#endif /* static */ +#endif /* emacs */ + +/* If your stack is a linked list of frames, you have to + provide an "address metric" ADDRESS_FUNCTION macro. */ + +#if defined (CRAY) && defined (CRAY_STACKSEG_END) +long i00afunc (); +#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg)) +#else +#define ADDRESS_FUNCTION(arg) &(arg) +#endif + +#if __STDC__ +typedef void *pointer; +#else +typedef char *pointer; +#endif + +#define NULL 0 + +/* Different portions of Emacs need to call different versions of + malloc. The Emacs executable needs alloca to call xmalloc, because + ordinary malloc isn't protected from input signals. On the other + hand, the utilities in lib-src need alloca to call malloc; some of + them are very simple, and don't have an xmalloc routine. + + Non-Emacs programs expect this to call use xmalloc. + + Callers below should use malloc. */ + +#ifndef emacs +#define malloc xmalloc +#endif +extern pointer malloc (); + +/* Define STACK_DIRECTION if you know the direction of stack + growth for your system; otherwise it will be automatically + deduced at run-time. + + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ + +#ifndef STACK_DIRECTION +#define STACK_DIRECTION 0 /* Direction unknown. */ +#endif + +#if STACK_DIRECTION != 0 + +#define STACK_DIR STACK_DIRECTION /* Known at compile-time. */ + +#else /* STACK_DIRECTION == 0; need run-time code. */ + +static int stack_dir; /* 1 or -1 once known. */ +#define STACK_DIR stack_dir + +static void +find_stack_direction () +{ + static char *addr = NULL; /* Address of first `dummy', once known. */ + auto char dummy; /* To get stack address. */ + + if (addr == NULL) + { /* Initial entry. */ + addr = ADDRESS_FUNCTION (dummy); + + find_stack_direction (); /* Recurse once. */ + } + else + { + /* Second entry. */ + if (ADDRESS_FUNCTION (dummy) > addr) + stack_dir = 1; /* Stack grew upward. */ + else + stack_dir = -1; /* Stack grew downward. */ + } +} + +#endif /* STACK_DIRECTION == 0 */ + +/* An "alloca header" is used to: + (a) chain together all alloca'ed blocks; + (b) keep track of stack depth. + + It is very important that sizeof(header) agree with malloc + alignment chunk size. The following default should work okay. */ + +#ifndef ALIGN_SIZE +#define ALIGN_SIZE sizeof(double) +#endif + +typedef union hdr +{ + char align[ALIGN_SIZE]; /* To force sizeof(header). */ + struct + { + union hdr *next; /* For chaining headers. */ + char *deep; /* For stack depth measure. */ + } h; +} header; + +static header *last_alloca_header = NULL; /* -> last alloca header. */ + +/* Return a pointer to at least SIZE bytes of storage, + which will be automatically reclaimed upon exit from + the procedure that called alloca. Originally, this space + was supposed to be taken from the current stack frame of the + caller, but that method cannot be made to work for some + implementations of C, for example under Gould's UTX/32. */ + +pointer +alloca (size) + unsigned size; +{ + auto char probe; /* Probes stack depth: */ + register char *depth = ADDRESS_FUNCTION (probe); + +#if STACK_DIRECTION == 0 + if (STACK_DIR == 0) /* Unknown growth direction. */ + find_stack_direction (); +#endif + + /* Reclaim garbage, defined as all alloca'd storage that + was allocated from deeper in the stack than currently. */ + + { + register header *hp; /* Traverses linked list. */ + + for (hp = last_alloca_header; hp != NULL;) + if ((STACK_DIR > 0 && hp->h.deep > depth) + || (STACK_DIR < 0 && hp->h.deep < depth)) + { + register header *np = hp->h.next; + + free ((pointer) hp); /* Collect garbage. */ + + hp = np; /* -> next header. */ + } + else + break; /* Rest are not deeper. */ + + last_alloca_header = hp; /* -> last valid storage. */ + } + + if (size == 0) + return NULL; /* No allocation required. */ + + /* Allocate combined header + user data storage. */ + + { + register pointer new = malloc (sizeof (header) + size); + /* Address of header. */ + + ((header *) new)->h.next = last_alloca_header; + ((header *) new)->h.deep = depth; + + last_alloca_header = (header *) new; + + /* User storage begins just after header. */ + + return (pointer) ((char *) new + sizeof (header)); + } +} + +#if defined (CRAY) && defined (CRAY_STACKSEG_END) + +#ifdef DEBUG_I00AFUNC +#include <stdio.h> +#endif + +#ifndef CRAY_STACK +#define CRAY_STACK +#ifndef CRAY2 +/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */ +struct stack_control_header + { + long shgrow:32; /* Number of times stack has grown. */ + long shaseg:32; /* Size of increments to stack. */ + long shhwm:32; /* High water mark of stack. */ + long shsize:32; /* Current size of stack (all segments). */ + }; + +/* The stack segment linkage control information occurs at + the high-address end of a stack segment. (The stack + grows from low addresses to high addresses.) The initial + part of the stack segment linkage control information is + 0200 (octal) words. This provides for register storage + for the routine which overflows the stack. */ + +struct stack_segment_linkage + { + long ss[0200]; /* 0200 overflow words. */ + long sssize:32; /* Number of words in this segment. */ + long ssbase:32; /* Offset to stack base. */ + long:32; + long sspseg:32; /* Offset to linkage control of previous + segment of stack. */ + long:32; + long sstcpt:32; /* Pointer to task common address block. */ + long sscsnm; /* Private control structure number for + microtasking. */ + long ssusr1; /* Reserved for user. */ + long ssusr2; /* Reserved for user. */ + long sstpid; /* Process ID for pid based multi-tasking. */ + long ssgvup; /* Pointer to multitasking thread giveup. */ + long sscray[7]; /* Reserved for Cray Research. */ + long ssa0; + long ssa1; + long ssa2; + long ssa3; + long ssa4; + long ssa5; + long ssa6; + long ssa7; + long sss0; + long sss1; + long sss2; + long sss3; + long sss4; + long sss5; + long sss6; + long sss7; + }; + +#else /* CRAY2 */ +/* The following structure defines the vector of words + returned by the STKSTAT library routine. */ +struct stk_stat + { + long now; /* Current total stack size. */ + long maxc; /* Amount of contiguous space which would + be required to satisfy the maximum + stack demand to date. */ + long high_water; /* Stack high-water mark. */ + long overflows; /* Number of stack overflow ($STKOFEN) calls. */ + long hits; /* Number of internal buffer hits. */ + long extends; /* Number of block extensions. */ + long stko_mallocs; /* Block allocations by $STKOFEN. */ + long underflows; /* Number of stack underflow calls ($STKRETN). */ + long stko_free; /* Number of deallocations by $STKRETN. */ + long stkm_free; /* Number of deallocations by $STKMRET. */ + long segments; /* Current number of stack segments. */ + long maxs; /* Maximum number of stack segments so far. */ + long pad_size; /* Stack pad size. */ + long current_address; /* Current stack segment address. */ + long current_size; /* Current stack segment size. This + number is actually corrupted by STKSTAT to + include the fifteen word trailer area. */ + long initial_address; /* Address of initial segment. */ + long initial_size; /* Size of initial segment. */ + }; + +/* The following structure describes the data structure which trails + any stack segment. I think that the description in 'asdef' is + out of date. I only describe the parts that I am sure about. */ + +struct stk_trailer + { + long this_address; /* Address of this block. */ + long this_size; /* Size of this block (does not include + this trailer). */ + long unknown2; + long unknown3; + long link; /* Address of trailer block of previous + segment. */ + long unknown5; + long unknown6; + long unknown7; + long unknown8; + long unknown9; + long unknown10; + long unknown11; + long unknown12; + long unknown13; + long unknown14; + }; + +#endif /* CRAY2 */ +#endif /* not CRAY_STACK */ + +#ifdef CRAY2 +/* Determine a "stack measure" for an arbitrary ADDRESS. + I doubt that "lint" will like this much. */ + +static long +i00afunc (long *address) +{ + struct stk_stat status; + struct stk_trailer *trailer; + long *block, size; + long result = 0; + + /* We want to iterate through all of the segments. The first + step is to get the stack status structure. We could do this + more quickly and more directly, perhaps, by referencing the + $LM00 common block, but I know that this works. */ + + STKSTAT (&status); + + /* Set up the iteration. */ + + trailer = (struct stk_trailer *) (status.current_address + + status.current_size + - 15); + + /* There must be at least one stack segment. Therefore it is + a fatal error if "trailer" is null. */ + + if (trailer == 0) + abort (); + + /* Discard segments that do not contain our argument address. */ + + while (trailer != 0) + { + block = (long *) trailer->this_address; + size = trailer->this_size; + if (block == 0 || size == 0) + abort (); + trailer = (struct stk_trailer *) trailer->link; + if ((block <= address) && (address < (block + size))) + break; + } + + /* Set the result to the offset in this segment and add the sizes + of all predecessor segments. */ + + result = address - block; + + if (trailer == 0) + { + return result; + } + + do + { + if (trailer->this_size <= 0) + abort (); + result += trailer->this_size; + trailer = (struct stk_trailer *) trailer->link; + } + while (trailer != 0); + + /* We are done. Note that if you present a bogus address (one + not in any segment), you will get a different number back, formed + from subtracting the address of the first block. This is probably + not what you want. */ + + return (result); +} + +#else /* not CRAY2 */ +/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP. + Determine the number of the cell within the stack, + given the address of the cell. The purpose of this + routine is to linearize, in some sense, stack addresses + for alloca. */ + +static long +i00afunc (long address) +{ + long stkl = 0; + + long size, pseg, this_segment, stack; + long result = 0; + + struct stack_segment_linkage *ssptr; + + /* Register B67 contains the address of the end of the + current stack segment. If you (as a subprogram) store + your registers on the stack and find that you are past + the contents of B67, you have overflowed the segment. + + B67 also points to the stack segment linkage control + area, which is what we are really interested in. */ + + stkl = CRAY_STACKSEG_END (); + ssptr = (struct stack_segment_linkage *) stkl; + + /* If one subtracts 'size' from the end of the segment, + one has the address of the first word of the segment. + + If this is not the first segment, 'pseg' will be + nonzero. */ + + pseg = ssptr->sspseg; + size = ssptr->sssize; + + this_segment = stkl - size; + + /* It is possible that calling this routine itself caused + a stack overflow. Discard stack segments which do not + contain the target address. */ + + while (!(this_segment <= address && address <= stkl)) + { +#ifdef DEBUG_I00AFUNC + fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl); +#endif + if (pseg == 0) + break; + stkl = stkl - pseg; + ssptr = (struct stack_segment_linkage *) stkl; + size = ssptr->sssize; + pseg = ssptr->sspseg; + this_segment = stkl - size; + } + + result = address - this_segment; + + /* If you subtract pseg from the current end of the stack, + you get the address of the previous stack segment's end. + This seems a little convoluted to me, but I'll bet you save + a cycle somewhere. */ + + while (pseg != 0) + { +#ifdef DEBUG_I00AFUNC + fprintf (stderr, "%011o %011o\n", pseg, size); +#endif + stkl = stkl - pseg; + ssptr = (struct stack_segment_linkage *) stkl; + size = ssptr->sssize; + pseg = ssptr->sspseg; + result += size; + } + return (result); +} + +#endif /* not CRAY2 */ +#endif /* CRAY */ + +#endif /* no alloca */ +#endif /* not GCC version 2 */ diff --git a/MISC/debflex.awk b/MISC/debflex.awk new file mode 100644 index 0000000..b1eda49 --- /dev/null +++ b/MISC/debflex.awk @@ -0,0 +1,119 @@ +# Clarify the flex debug trace by substituting first line of each rule. +# Francois Pinard <pinard@iro.umontreal.ca>, July 1990. +# +# Rewritten to process correctly \n's in scanner input. +# BEGIN section modified to correct a collection of rules. +# Michal Jaegermann <michal@phys.ualberta.ca>, December 1993 +# +# Sample usage: +# flex -d PROGRAM.l +# gcc -o PROGRAM PROGRAM.c -lfl +# PROGRAM 2>&1 | gawk -f debflex.awk PROGRAM.l +# +# (VP's note: this script presently does not work with either "old" or +# "new" awk; fixes so it does will be welcome) + +BEGIN { + # Insure proper usage. + + if (ARGC != 2) { + print "usage: gawk -f debflex.awk FLEX_SOURCE <DEBUG_OUTPUT"; + exit (1); + } + + # Remove and save the name of flex source. + + source = ARGV[1]; + ARGC--; + + # Swallow the flex source file. + + line = 0; + section = 1; + while (getline <source) { + + # Count the lines. + + line++; + + # Count the sections. When encountering section 3, + # break out of the awk BEGIN block. + + if (match ($0, /^%%/)) { + section++; + if (section == 3) { + break; + } + } + else { + # Only the lines in section 2 which do not begin in a + # tab or space might be referred to by the flex debug + # trace. Save only those lines. + + if (section == 2 && match ($0, /^[^ \t]/)) { + rules[line] = $0; + } + } + } + dashes = "-----------------------------------------------------------"; + collect = ""; + line = 0; +} + +# collect complete rule output from a scanner +$0 !~ /^--/ { + collect = collect "\n" $0; + next; +} +# otherwise we have a new rule - process what we got so far +{ + process(); +} +# and the same thing if we hit EOF +END { + process(); +} + +function process() { + + # splitting this way we loose some double dashes and + # left parentheses from echoed input - a small price to pay + n = split(collect, field, "\n--|[(]"); + + # this loop kicks in only when we already collected something + for (i = 1; i <= n; i++) { + if (0 != line) { + # we do not care for traces of newlines. + if (0 == match(field[i], /\"\n+\"[)]/)) { + if (rules[line]) { + text = field[i]; + while ( ++i <= n) { + text = text field[i]; + } + printf("%s:%d: %-8s -- %s\n", + source, line, text, rules[line]); + } + else { + print; + printf "%s:%d: *** No such rule.\n", source, line; + } + } + line = 0; + break; + } + if ("" != field[i]) { + if ("end of buffer or a NUL)" == field[i]) { + print dashes; # Simplify trace of buffer reloads + continue; + } + if (match(field[i], /accepting rule at line /)) { + # force interpretation of line as a number + line = 0 + substr(field[i], RLENGTH); + continue; + } + # echo everything else + printf("--%s\n", field[i]); + } + } + collect = "\n" $0; # ... and start next trace +} diff --git a/MISC/fastwc/README b/MISC/fastwc/README new file mode 100644 index 0000000..0dd3afe --- /dev/null +++ b/MISC/fastwc/README @@ -0,0 +1,56 @@ +This directory contains some examples illustrating techniques for extracting +high-performance from flex scanners. Each program implements a simplified +version of the Unix "wc" tool: read text from stdin and print the number of +characters, words, and lines present in the text. All programs were compiled +using gcc (version unavailable, sorry) with the -O flag, and run on a +SPARCstation 1+. The input used was a PostScript file, mainly containing +figures, with the following "wc" counts: + + lines words characters + 214217 635954 2592172 + + +The basic principles illustrated by these programs are: + + - match as much text with each rule as possible + - adding rules does not slow you down! + - avoid backing up + +and the big caveat that comes with them is: + + - you buy performance with decreased maintainability; make + sure you really need it before applying the above techniques. + +See the "Performance Considerations" section of flexdoc for more +details regarding these principles. + + +The different versions of "wc": + + mywc.c + a simple but fairly efficient C version + + wc1.l a naive flex "wc" implementation + + wc2.l somewhat faster; adds rules to match multiple tokens at once + + wc3.l faster still; adds more rules to match longer runs of tokens + + wc4.l fastest; still more rules added; hard to do much better + using flex (or, I suspect, hand-coding) + + wc5.l identical to wc3.l except one rule has been slightly + shortened, introducing backing-up + +Timing results (all times in user CPU seconds): + + program time notes + ------- ---- ----- + wc1 16.4 default flex table compression (= -Cem) + wc1 6.7 -Cf compression option + /bin/wc 5.8 Sun's standard "wc" tool + mywc 4.6 simple but better C implementation! + wc2 4.6 as good as C implementation; built using -Cf + wc3 3.8 -Cf + wc4 3.3 -Cf + wc5 5.7 -Cf; ouch, backing up is expensive diff --git a/MISC/fastwc/mywc.c b/MISC/fastwc/mywc.c new file mode 100644 index 0000000..92e5a36 --- /dev/null +++ b/MISC/fastwc/mywc.c @@ -0,0 +1,26 @@ +/* A simple but fairly efficient C version of the Unix "wc" tool */ + +#include <stdio.h> +#include <ctype.h> + +main() +{ + register int c, cc = 0, wc = 0, lc = 0; + FILE *f = stdin; + + while ((c = getc(f)) != EOF) { + ++cc; + if (isgraph(c)) { + ++wc; + do { + c = getc(f); + if (c == EOF) + goto done; + ++cc; + } while (isgraph(c)); + } + if (c == '\n') + ++lc; + } +done: printf( "%8d%8d%8d\n", lc, wc, cc ); +} diff --git a/MISC/fastwc/wc1.l b/MISC/fastwc/wc1.l new file mode 100644 index 0000000..d6696bc --- /dev/null +++ b/MISC/fastwc/wc1.l @@ -0,0 +1,18 @@ +/* First cut at a flex-based "wc" tool. */ + +ws [ \t] +nonws [^ \t\n] + +%% + int cc = 0, wc = 0, lc = 0; + +{nonws}+ cc += yyleng; ++wc; + +{ws}+ cc += yyleng; + +\n ++lc; ++cc; + +<<EOF>> { + printf( "%8d %8d %8d\n", lc, wc, cc ); + yyterminate(); + } diff --git a/MISC/fastwc/wc2.l b/MISC/fastwc/wc2.l new file mode 100644 index 0000000..bd63cd4 --- /dev/null +++ b/MISC/fastwc/wc2.l @@ -0,0 +1,20 @@ +/* Somewhat faster "wc" tool: match more text with each rule */ + +ws [ \t] +nonws [^ \t\n] +word {ws}*{nonws}+ + +%% + int cc = 0, wc = 0, lc = 0; + +{word}{ws}* cc += yyleng; ++wc; +{word}{ws}*\n cc += yyleng; ++wc; ++lc; + +{ws}+ cc += yyleng; + +\n+ cc += yyleng; lc += yyleng; + +<<EOF>> { + printf( "%8d %8d %8d\n", lc, wc, cc ); + yyterminate(); + } diff --git a/MISC/fastwc/wc3.l b/MISC/fastwc/wc3.l new file mode 100644 index 0000000..7c5f2e2 --- /dev/null +++ b/MISC/fastwc/wc3.l @@ -0,0 +1,24 @@ +/* Somewhat faster still: potentially match a lot of text with each rule */ + +ws [ \t] +nonws [^ \t\n] +word {ws}*{nonws}+ +words {word}{ws}+ + +%% + int cc = 0, wc = 0, lc = 0; + +{word}{ws}* cc += yyleng; ++wc; +{word}{ws}*\n cc += yyleng; ++wc; ++lc; +{words}{word}{ws}* cc += yyleng; wc += 2; +{words}{2}{word}{ws}* cc += yyleng; wc += 3; +{words}{3}{word}{ws}* cc += yyleng; wc += 4; + +{ws}+ cc += yyleng; + +\n+ cc += yyleng; lc += yyleng; + +<<EOF>> { + printf( "%8d %8d %8d\n", lc, wc, cc ); + yyterminate(); + } diff --git a/MISC/fastwc/wc4.l b/MISC/fastwc/wc4.l new file mode 100644 index 0000000..cbe56f6 --- /dev/null +++ b/MISC/fastwc/wc4.l @@ -0,0 +1,27 @@ +/* Fastest version of wc: add rules to pick up newlines, too */ + +ws [ \t] +nonws [^ \t\n] +word {ws}*{nonws}+ +words {word}{ws}+ + +%% + int cc = 0, wc = 0, lc = 0; + +{word}{ws}* ++wc; cc += yyleng; +{word}{ws}*\n ++wc; cc += yyleng; ++lc; +{words}{word}{ws}* wc += 2; cc += yyleng; +{words}{word}{ws}*\n wc += 2; cc += yyleng; ++lc; +{words}{2}{word}{ws}* wc += 3; cc += yyleng; +{words}{2}{word}{ws}*\n wc += 3; cc += yyleng; ++lc; +{words}{3}{word}{ws}* wc += 4; cc += yyleng; +{words}{3}{word}{ws}*\n wc += 4; cc += yyleng; ++lc; + +{ws}+ cc += yyleng; + +\n+ cc += yyleng; lc += yyleng; + +<<EOF>> { + printf( "%8d %8d %8d\n", lc, wc, cc ); + yyterminate(); + } diff --git a/MISC/fastwc/wc5.l b/MISC/fastwc/wc5.l new file mode 100644 index 0000000..8fe17b6 --- /dev/null +++ b/MISC/fastwc/wc5.l @@ -0,0 +1,24 @@ +/* Oops; slight change from wc3.l introduces backtracking */ + +ws [ \t] +nonws [^ \t\n] +word {ws}*{nonws}+ +words {word}{ws}+ + +%% + int cc = 0, wc = 0, lc = 0; + +{word}{ws}* cc += yyleng; ++wc; +{word}{ws}*\n cc += yyleng; ++wc; ++lc; +{words}{word} cc += yyleng; wc += 2; /* oops */ +{words}{2}{word}{ws}* cc += yyleng; wc += 3; +{words}{3}{word}{ws}* cc += yyleng; wc += 4; + +{ws}+ cc += yyleng; + +\n+ cc += yyleng; lc += yyleng; + +<<EOF>> { + printf( "%8d %8d %8d\n", lc, wc, cc ); + yyterminate(); + } diff --git a/MISC/flex.man b/MISC/flex.man new file mode 100644 index 0000000..d41f5ba --- /dev/null +++ b/MISC/flex.man @@ -0,0 +1,3696 @@ + + + +FLEX(1) USER COMMANDS FLEX(1) + + + +NAME + flex - fast lexical analyzer generator + +SYNOPSIS + flex [-bcdfhilnpstvwBFILTV78+? -C[aefFmr] -ooutput -Pprefix + -Sskeleton] [--help --version] [filename ...] + +OVERVIEW + This manual describes flex, a tool for generating programs + that perform pattern-matching on text. The manual includes + both tutorial and reference sections: + + Description + a brief overview of the tool + + Some Simple Examples + + Format Of The Input File + + Patterns + the extended regular expressions used by flex + + How The Input Is Matched + the rules for determining what has been matched + + Actions + how to specify what to do when a pattern is matched + + The Generated Scanner + details regarding the scanner that flex produces; + how to control the input source + + Start Conditions + introducing context into your scanners, and + managing "mini-scanners" + + Multiple Input Buffers + how to manipulate multiple input sources; how to + scan from strings instead of files + + End-of-file Rules + special rules for matching the end of the input + + Miscellaneous Macros + a summary of macros available to the actions + + Values Available To The User + a summary of values available to the actions + + Interfacing With Yacc + connecting flex scanners together with yacc parsers + + + + +Version 2.5 Last change: April 1995 1 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + Options + flex command-line options, and the "%option" + directive + + Performance Considerations + how to make your scanner go as fast as possible + + Generating C++ Scanners + the (experimental) facility for generating C++ + scanner classes + + Incompatibilities With Lex And POSIX + how flex differs from AT&T lex and the POSIX lex + standard + + Diagnostics + those error messages produced by flex (or scanners + it generates) whose meanings might not be apparent + + Files + files used by flex + + Deficiencies / Bugs + known problems with flex + + See Also + other documentation, related tools + + Author + includes contact information + + +DESCRIPTION + flex is a tool for generating scanners: programs which + recognized lexical patterns in text. flex reads the given + input files, or its standard input if no file names are + given, for a description of a scanner to generate. The + description is in the form of pairs of regular expressions + and C code, called rules. flex generates as output a C + source file, lex.yy.c, which defines a routine yylex(). This + file is compiled and linked with the -lfl library to produce + an executable. When the executable is run, it analyzes its + input for occurrences of the regular expressions. Whenever + it finds one, it executes the corresponding C code. + +SOME SIMPLE EXAMPLES + First some simple examples to get the flavor of how one uses + flex. The following flex input specifies a scanner which + whenever it encounters the string "username" will replace it + with the user's login name: + + %% + + + +Version 2.5 Last change: April 1995 2 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + username printf( "%s", getlogin() ); + + By default, any text not matched by a flex scanner is copied + to the output, so the net effect of this scanner is to copy + its input file to its output with each occurrence of "user- + name" expanded. In this input, there is just one rule. + "username" is the pattern and the "printf" is the action. + The "%%" marks the beginning of the rules. + + Here's another simple example: + + int num_lines = 0, num_chars = 0; + + %% + \n ++num_lines; ++num_chars; + . ++num_chars; + + %% + main() + { + yylex(); + printf( "# of lines = %d, # of chars = %d\n", + num_lines, num_chars ); + } + + This scanner counts the number of characters and the number + of lines in its input (it produces no output other than the + final report on the counts). The first line declares two + globals, "num_lines" and "num_chars", which are accessible + both inside yylex() and in the main() routine declared after + the second "%%". There are two rules, one which matches a + newline ("\n") and increments both the line count and the + character count, and one which matches any character other + than a newline (indicated by the "." regular expression). + + A somewhat more complicated example: + + /* scanner for a toy Pascal-like language */ + + %{ + /* need this for the call to atof() below */ + #include <math.h> + %} + + DIGIT [0-9] + ID [a-z][a-z0-9]* + + %% + + {DIGIT}+ { + printf( "An integer: %s (%d)\n", yytext, + atoi( yytext ) ); + + + +Version 2.5 Last change: April 1995 3 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + } + + {DIGIT}+"."{DIGIT}* { + printf( "A float: %s (%g)\n", yytext, + atof( yytext ) ); + } + + if|then|begin|end|procedure|function { + printf( "A keyword: %s\n", yytext ); + } + + {ID} printf( "An identifier: %s\n", yytext ); + + "+"|"-"|"*"|"/" printf( "An operator: %s\n", yytext ); + + "{"[^}\n]*"}" /* eat up one-line comments */ + + [ \t\n]+ /* eat up whitespace */ + + . printf( "Unrecognized character: %s\n", yytext ); + + %% + + main( argc, argv ) + int argc; + char **argv; + { + ++argv, --argc; /* skip over program name */ + if ( argc > 0 ) + yyin = fopen( argv[0], "r" ); + else + yyin = stdin; + + yylex(); + } + + This is the beginnings of a simple scanner for a language + like Pascal. It identifies different types of tokens and + reports on what it has seen. + + The details of this example will be explained in the follow- + ing sections. + +FORMAT OF THE INPUT FILE + The flex input file consists of three sections, separated by + a line with just %% in it: + + definitions + %% + rules + %% + user code + + + +Version 2.5 Last change: April 1995 4 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + The definitions section contains declarations of simple name + definitions to simplify the scanner specification, and + declarations of start conditions, which are explained in a + later section. + + Name definitions have the form: + + name definition + + The "name" is a word beginning with a letter or an under- + score ('_') followed by zero or more letters, digits, '_', + or '-' (dash). The definition is taken to begin at the + first non-white-space character following the name and con- + tinuing to the end of the line. The definition can subse- + quently be referred to using "{name}", which will expand to + "(definition)". For example, + + DIGIT [0-9] + ID [a-z][a-z0-9]* + + defines "DIGIT" to be a regular expression which matches a + single digit, and "ID" to be a regular expression which + matches a letter followed by zero-or-more letters-or-digits. + A subsequent reference to + + {DIGIT}+"."{DIGIT}* + + is identical to + + ([0-9])+"."([0-9])* + + and matches one-or-more digits followed by a '.' followed by + zero-or-more digits. + + The rules section of the flex input contains a series of + rules of the form: + + pattern action + + where the pattern must be unindented and the action must + begin on the same line. + + See below for a further description of patterns and actions. + + Finally, the user code section is simply copied to lex.yy.c + verbatim. It is used for companion routines which call or + are called by the scanner. The presence of this section is + optional; if it is missing, the second %% in the input file + may be skipped, too. + + In the definitions and rules sections, any indented text or + text enclosed in %{ and %} is copied verbatim to the output + + + +Version 2.5 Last change: April 1995 5 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + (with the %{}'s removed). The %{}'s must appear unindented + on lines by themselves. + + In the rules section, any indented or %{} text appearing + before the first rule may be used to declare variables which + are local to the scanning routine and (after the declara- + tions) code which is to be executed whenever the scanning + routine is entered. Other indented or %{} text in the rule + section is still copied to the output, but its meaning is + not well-defined and it may well cause compile-time errors + (this feature is present for POSIX compliance; see below for + other such features). + + In the definitions section (but not in the rules section), + an unindented comment (i.e., a line beginning with "/*") is + also copied verbatim to the output up to the next "*/". + +PATTERNS + The patterns in the input are written using an extended set + of regular expressions. These are: + + x match the character 'x' + . any character (byte) except newline + [xyz] a "character class"; in this case, the pattern + matches either an 'x', a 'y', or a 'z' + [abj-oZ] a "character class" with a range in it; matches + an 'a', a 'b', any letter from 'j' through 'o', + or a 'Z' + [^A-Z] a "negated character class", i.e., any character + but those in the class. In this case, any + character EXCEPT an uppercase letter. + [^A-Z\n] any character EXCEPT an uppercase letter or + a newline + r* zero or more r's, where r is any regular expression + r+ one or more r's + r? zero or one r's (that is, "an optional r") + r{2,5} anywhere from two to five r's + r{2,} two or more r's + r{4} exactly 4 r's + {name} the expansion of the "name" definition + (see above) + "[xyz]\"foo" + the literal string: [xyz]"foo + \X if X is an 'a', 'b', 'f', 'n', 'r', 't', or 'v', + then the ANSI-C interpretation of \x. + Otherwise, a literal 'X' (used to escape + operators such as '*') + \0 a NUL character (ASCII code 0) + \123 the character with octal value 123 + \x2a the character with hexadecimal value 2a + (r) match an r; parentheses are used to override + precedence (see below) + + + +Version 2.5 Last change: April 1995 6 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + rs the regular expression r followed by the + regular expression s; called "concatenation" + + + r|s either an r or an s + + + r/s an r but only if it is followed by an s. The + text matched by s is included when determining + whether this rule is the "longest match", + but is then returned to the input before + the action is executed. So the action only + sees the text matched by r. This type + of pattern is called trailing context". + (There are some combinations of r/s that flex + cannot match correctly; see notes in the + Deficiencies / Bugs section below regarding + "dangerous trailing context".) + ^r an r, but only at the beginning of a line (i.e., + which just starting to scan, or right after a + newline has been scanned). + r$ an r, but only at the end of a line (i.e., just + before a newline). Equivalent to "r/\n". + + Note that flex's notion of "newline" is exactly + whatever the C compiler used to compile flex + interprets '\n' as; in particular, on some DOS + systems you must either filter out \r's in the + input yourself, or explicitly use r/\r\n for "r$". + + + <s>r an r, but only in start condition s (see + below for discussion of start conditions) + <s1,s2,s3>r + same, but in any of start conditions s1, + s2, or s3 + <*>r an r in any start condition, even an exclusive one. + + + <<EOF>> an end-of-file + <s1,s2><<EOF>> + an end-of-file when in start condition s1 or s2 + + Note that inside of a character class, all regular expres- + sion operators lose their special meaning except escape + ('\') and the character class operators, '-', ']', and, at + the beginning of the class, '^'. + + The regular expressions listed above are grouped according + to precedence, from highest precedence at the top to lowest + at the bottom. Those grouped together have equal pre- + cedence. For example, + + + +Version 2.5 Last change: April 1995 7 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + foo|bar* + + is the same as + + (foo)|(ba(r*)) + + since the '*' operator has higher precedence than concatena- + tion, and concatenation higher than alternation ('|'). This + pattern therefore matches either the string "foo" or the + string "ba" followed by zero-or-more r's. To match "foo" or + zero-or-more "bar"'s, use: + + foo|(bar)* + + and to match zero-or-more "foo"'s-or-"bar"'s: + + (foo|bar)* + + + In addition to characters and ranges of characters, charac- + ter classes can also contain character class expressions. + These are expressions enclosed inside [: and :] delimiters + (which themselves must appear between the '[' and ']' of the + character class; other elements may occur inside the charac- + ter class, too). The valid expressions are: + + [:alnum:] [:alpha:] [:blank:] + [:cntrl:] [:digit:] [:graph:] + [:lower:] [:print:] [:punct:] + [:space:] [:upper:] [:xdigit:] + + These expressions all designate a set of characters + equivalent to the corresponding standard C isXXX function. + For example, [:alnum:] designates those characters for which + isalnum() returns true - i.e., any alphabetic or numeric. + Some systems don't provide isblank(), so flex defines + [:blank:] as a blank or a tab. + + For example, the following character classes are all + equivalent: + + [[:alnum:]] + [[:alpha:][:digit:] + [[:alpha:]0-9] + [a-zA-Z0-9] + + If your scanner is case-insensitive (the -i flag), then + [:upper:] and [:lower:] are equivalent to [:alpha:]. + + Some notes on patterns: + + - A negated character class such as the example "[^A-Z]" + + + +Version 2.5 Last change: April 1995 8 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + above will match a newline unless "\n" (or an + equivalent escape sequence) is one of the characters + explicitly present in the negated character class + (e.g., "[^A-Z\n]"). This is unlike how many other reg- + ular expression tools treat negated character classes, + but unfortunately the inconsistency is historically + entrenched. Matching newlines means that a pattern + like [^"]* can match the entire input unless there's + another quote in the input. + + - A rule can have at most one instance of trailing con- + text (the '/' operator or the '$' operator). The start + condition, '^', and "<<EOF>>" patterns can only occur + at the beginning of a pattern, and, as well as with '/' + and '$', cannot be grouped inside parentheses. A '^' + which does not occur at the beginning of a rule or a + '$' which does not occur at the end of a rule loses its + special properties and is treated as a normal charac- + ter. + + The following are illegal: + + foo/bar$ + <sc1>foo<sc2>bar + + Note that the first of these, can be written + "foo/bar\n". + + The following will result in '$' or '^' being treated + as a normal character: + + foo|(bar$) + foo|^bar + + If what's wanted is a "foo" or a bar-followed-by-a- + newline, the following could be used (the special '|' + action is explained below): + + foo | + bar$ /* action goes here */ + + A similar trick will work for matching a foo or a bar- + at-the-beginning-of-a-line. + +HOW THE INPUT IS MATCHED + When the generated scanner is run, it analyzes its input + looking for strings which match any of its patterns. If it + finds more than one match, it takes the one matching the + most text (for trailing context rules, this includes the + length of the trailing part, even though it will then be + returned to the input). If it finds two or more matches of + the same length, the rule listed first in the flex input + + + +Version 2.5 Last change: April 1995 9 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + file is chosen. + + Once the match is determined, the text corresponding to the + match (called the token) is made available in the global + character pointer yytext, and its length in the global + integer yyleng. The action corresponding to the matched pat- + tern is then executed (a more detailed description of + actions follows), and then the remaining input is scanned + for another match. + + If no match is found, then the default rule is executed: the + next character in the input is considered matched and copied + to the standard output. Thus, the simplest legal flex input + is: + + %% + + which generates a scanner that simply copies its input (one + character at a time) to its output. + + Note that yytext can be defined in two different ways: + either as a character pointer or as a character array. You + can control which definition flex uses by including one of + the special directives %pointer or %array in the first + (definitions) section of your flex input. The default is + %pointer, unless you use the -l lex compatibility option, in + which case yytext will be an array. The advantage of using + %pointer is substantially faster scanning and no buffer + overflow when matching very large tokens (unless you run out + of dynamic memory). The disadvantage is that you are res- + tricted in how your actions can modify yytext (see the next + section), and calls to the unput() function destroys the + present contents of yytext, which can be a considerable + porting headache when moving between different lex versions. + + The advantage of %array is that you can then modify yytext + to your heart's content, and calls to unput() do not destroy + yytext (see below). Furthermore, existing lex programs + sometimes access yytext externally using declarations of the + form: + extern char yytext[]; + This definition is erroneous when used with %pointer, but + correct for %array. + + %array defines yytext to be an array of YYLMAX characters, + which defaults to a fairly large value. You can change the + size by simply #define'ing YYLMAX to a different value in + the first section of your flex input. As mentioned above, + with %pointer yytext grows dynamically to accommodate large + tokens. While this means your %pointer scanner can accommo- + date very large tokens (such as matching entire blocks of + comments), bear in mind that each time the scanner must + + + +Version 2.5 Last change: April 1995 10 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + resize yytext it also must rescan the entire token from the + beginning, so matching such tokens can prove slow. yytext + presently does not dynamically grow if a call to unput() + results in too much text being pushed back; instead, a run- + time error results. + + Also note that you cannot use %array with C++ scanner + classes (the c++ option; see below). + +ACTIONS + Each pattern in a rule has a corresponding action, which can + be any arbitrary C statement. The pattern ends at the first + non-escaped whitespace character; the remainder of the line + is its action. If the action is empty, then when the pat- + tern is matched the input token is simply discarded. For + example, here is the specification for a program which + deletes all occurrences of "zap me" from its input: + + %% + "zap me" + + (It will copy all other characters in the input to the out- + put since they will be matched by the default rule.) + + Here is a program which compresses multiple blanks and tabs + down to a single blank, and throws away whitespace found at + the end of a line: + + %% + [ \t]+ putchar( ' ' ); + [ \t]+$ /* ignore this token */ + + + If the action contains a '{', then the action spans till the + balancing '}' is found, and the action may cross multiple + lines. flex knows about C strings and comments and won't be + fooled by braces found within them, but also allows actions + to begin with %{ and will consider the action to be all the + text up to the next %} (regardless of ordinary braces inside + the action). + + An action consisting solely of a vertical bar ('|') means + "same as the action for the next rule." See below for an + illustration. + + Actions can include arbitrary C code, including return + statements to return a value to whatever routine called + yylex(). Each time yylex() is called it continues processing + tokens from where it last left off until it either reaches + the end of the file or executes a return. + + + + + +Version 2.5 Last change: April 1995 11 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + Actions are free to modify yytext except for lengthening it + (adding characters to its end--these will overwrite later + characters in the input stream). This however does not + apply when using %array (see above); in that case, yytext + may be freely modified in any way. + + Actions are free to modify yyleng except they should not do + so if the action also includes use of yymore() (see below). + + There are a number of special directives which can be + included within an action: + + - ECHO copies yytext to the scanner's output. + + - BEGIN followed by the name of a start condition places + the scanner in the corresponding start condition (see + below). + + - REJECT directs the scanner to proceed on to the "second + best" rule which matched the input (or a prefix of the + input). The rule is chosen as described above in "How + the Input is Matched", and yytext and yyleng set up + appropriately. It may either be one which matched as + much text as the originally chosen rule but came later + in the flex input file, or one which matched less text. + For example, the following will both count the words in + the input and call the routine special() whenever + "frob" is seen: + + int word_count = 0; + %% + + frob special(); REJECT; + [^ \t\n]+ ++word_count; + + Without the REJECT, any "frob"'s in the input would not + be counted as words, since the scanner normally exe- + cutes only one action per token. Multiple REJECT's are + allowed, each one finding the next best choice to the + currently active rule. For example, when the following + scanner scans the token "abcd", it will write "abcdab- + caba" to the output: + + %% + a | + ab | + abc | + abcd ECHO; REJECT; + .|\n /* eat up any unmatched character */ + + (The first three rules share the fourth's action since + they use the special '|' action.) REJECT is a + + + +Version 2.5 Last change: April 1995 12 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + particularly expensive feature in terms of scanner per- + formance; if it is used in any of the scanner's actions + it will slow down all of the scanner's matching. + Furthermore, REJECT cannot be used with the -Cf or -CF + options (see below). + + Note also that unlike the other special actions, REJECT + is a branch; code immediately following it in the + action will not be executed. + + - yymore() tells the scanner that the next time it + matches a rule, the corresponding token should be + appended onto the current value of yytext rather than + replacing it. For example, given the input "mega- + kludge" the following will write "mega-mega-kludge" to + the output: + + %% + mega- ECHO; yymore(); + kludge ECHO; + + First "mega-" is matched and echoed to the output. + Then "kludge" is matched, but the previous "mega-" is + still hanging around at the beginning of yytext so the + ECHO for the "kludge" rule will actually write "mega- + kludge". + + Two notes regarding use of yymore(). First, yymore() depends + on the value of yyleng correctly reflecting the size of the + current token, so you must not modify yyleng if you are + using yymore(). Second, the presence of yymore() in the + scanner's action entails a minor performance penalty in the + scanner's matching speed. + + - yyless(n) returns all but the first n characters of the + current token back to the input stream, where they will + be rescanned when the scanner looks for the next match. + yytext and yyleng are adjusted appropriately (e.g., + yyleng will now be equal to n ). For example, on the + input "foobar" the following will write out "foobar- + bar": + + %% + foobar ECHO; yyless(3); + [a-z]+ ECHO; + + An argument of 0 to yyless will cause the entire + current input string to be scanned again. Unless + you've changed how the scanner will subsequently pro- + cess its input (using BEGIN, for example), this will + result in an endless loop. + + + + +Version 2.5 Last change: April 1995 13 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + Note that yyless is a macro and can only be used in the flex + input file, not from other source files. + + - unput(c) puts the character c back onto the input + stream. It will be the next character scanned. The + following action will take the current token and cause + it to be rescanned enclosed in parentheses. + + { + int i; + /* Copy yytext because unput() trashes yytext */ + char *yycopy = strdup( yytext ); + unput( ')' ); + for ( i = yyleng - 1; i >= 0; --i ) + unput( yycopy[i] ); + unput( '(' ); + free( yycopy ); + } + + Note that since each unput() puts the given character + back at the beginning of the input stream, pushing back + strings must be done back-to-front. + + An important potential problem when using unput() is that if + you are using %pointer (the default), a call to unput() des- + troys the contents of yytext, starting with its rightmost + character and devouring one character to the left with each + call. If you need the value of yytext preserved after a + call to unput() (as in the above example), you must either + first copy it elsewhere, or build your scanner using %array + instead (see How The Input Is Matched). + + Finally, note that you cannot put back EOF to attempt to + mark the input stream with an end-of-file. + + - input() reads the next character from the input stream. + For example, the following is one way to eat up C com- + ments: + + %% + "/*" { + register int c; + + for ( ; ; ) + { + while ( (c = input()) != '*' && + c != EOF ) + ; /* eat up text of comment */ + + if ( c == '*' ) + { + while ( (c = input()) == '*' ) + + + +Version 2.5 Last change: April 1995 14 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + ; + if ( c == '/' ) + break; /* found the end */ + } + + if ( c == EOF ) + { + error( "EOF in comment" ); + break; + } + } + } + + (Note that if the scanner is compiled using C++, then + input() is instead referred to as yyinput(), in order + to avoid a name clash with the C++ stream by the name + of input.) + + - YY_FLUSH_BUFFER flushes the scanner's internal buffer + so that the next time the scanner attempts to match a + token, it will first refill the buffer using YY_INPUT + (see The Generated Scanner, below). This action is a + special case of the more general yy_flush_buffer() + function, described below in the section Multiple Input + Buffers. + + - yyterminate() can be used in lieu of a return statement + in an action. It terminates the scanner and returns a + 0 to the scanner's caller, indicating "all done". By + default, yyterminate() is also called when an end-of- + file is encountered. It is a macro and may be rede- + fined. + +THE GENERATED SCANNER + The output of flex is the file lex.yy.c, which contains the + scanning routine yylex(), a number of tables used by it for + matching tokens, and a number of auxiliary routines and mac- + ros. By default, yylex() is declared as follows: + + int yylex() + { + ... various definitions and the actions in here ... + } + + (If your environment supports function prototypes, then it + will be "int yylex( void )".) This definition may be + changed by defining the "YY_DECL" macro. For example, you + could use: + + #define YY_DECL float lexscan( a, b ) float a, b; + + to give the scanning routine the name lexscan, returning a + + + +Version 2.5 Last change: April 1995 15 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + float, and taking two floats as arguments. Note that if you + give arguments to the scanning routine using a K&R- + style/non-prototyped function declaration, you must ter- + minate the definition with a semi-colon (;). + + Whenever yylex() is called, it scans tokens from the global + input file yyin (which defaults to stdin). It continues + until it either reaches an end-of-file (at which point it + returns the value 0) or one of its actions executes a return + statement. + + If the scanner reaches an end-of-file, subsequent calls are + undefined unless either yyin is pointed at a new input file + (in which case scanning continues from that file), or yyres- + tart() is called. yyrestart() takes one argument, a FILE * + pointer (which can be nil, if you've set up YY_INPUT to scan + from a source other than yyin), and initializes yyin for + scanning from that file. Essentially there is no difference + between just assigning yyin to a new input file or using + yyrestart() to do so; the latter is available for compati- + bility with previous versions of flex, and because it can be + used to switch input files in the middle of scanning. It + can also be used to throw away the current input buffer, by + calling it with an argument of yyin; but better is to use + YY_FLUSH_BUFFER (see above). Note that yyrestart() does not + reset the start condition to INITIAL (see Start Conditions, + below). + + If yylex() stops scanning due to executing a return state- + ment in one of the actions, the scanner may then be called + again and it will resume scanning where it left off. + + By default (and for purposes of efficiency), the scanner + uses block-reads rather than simple getc() calls to read + characters from yyin. The nature of how it gets its input + can be controlled by defining the YY_INPUT macro. + YY_INPUT's calling sequence is + "YY_INPUT(buf,result,max_size)". Its action is to place up + to max_size characters in the character array buf and return + in the integer variable result either the number of charac- + ters read or the constant YY_NULL (0 on Unix systems) to + indicate EOF. The default YY_INPUT reads from the global + file-pointer "yyin". + + A sample definition of YY_INPUT (in the definitions section + of the input file): + + %{ + #define YY_INPUT(buf,result,max_size) \ + { \ + int c = getchar(); \ + result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \ + + + +Version 2.5 Last change: April 1995 16 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + } + %} + + This definition will change the input processing to occur + one character at a time. + + When the scanner receives an end-of-file indication from + YY_INPUT, it then checks the yywrap() function. If yywrap() + returns false (zero), then it is assumed that the function + has gone ahead and set up yyin to point to another input + file, and scanning continues. If it returns true (non- + zero), then the scanner terminates, returning 0 to its + caller. Note that in either case, the start condition + remains unchanged; it does not revert to INITIAL. + + If you do not supply your own version of yywrap(), then you + must either use %option noyywrap (in which case the scanner + behaves as though yywrap() returned 1), or you must link + with -lfl to obtain the default version of the routine, + which always returns 1. + + Three routines are available for scanning from in-memory + buffers rather than files: yy_scan_string(), + yy_scan_bytes(), and yy_scan_buffer(). See the discussion of + them below in the section Multiple Input Buffers. + + The scanner writes its ECHO output to the yyout global + (default, stdout), which may be redefined by the user simply + by assigning it to some other FILE pointer. + +START CONDITIONS + flex provides a mechanism for conditionally activating + rules. Any rule whose pattern is prefixed with "<sc>" will + only be active when the scanner is in the start condition + named "sc". For example, + + <STRING>[^"]* { /* eat up the string body ... */ + ... + } + + will be active only when the scanner is in the "STRING" + start condition, and + + <INITIAL,STRING,QUOTE>\. { /* handle an escape ... */ + ... + } + + will be active only when the current start condition is + either "INITIAL", "STRING", or "QUOTE". + + Start conditions are declared in the definitions (first) + section of the input using unindented lines beginning with + + + +Version 2.5 Last change: April 1995 17 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + either %s or %x followed by a list of names. The former + declares inclusive start conditions, the latter exclusive + start conditions. A start condition is activated using the + BEGIN action. Until the next BEGIN action is executed, + rules with the given start condition will be active and + rules with other start conditions will be inactive. If the + start condition is inclusive, then rules with no start con- + ditions at all will also be active. If it is exclusive, + then only rules qualified with the start condition will be + active. A set of rules contingent on the same exclusive + start condition describe a scanner which is independent of + any of the other rules in the flex input. Because of this, + exclusive start conditions make it easy to specify "mini- + scanners" which scan portions of the input that are syntac- + tically different from the rest (e.g., comments). + + If the distinction between inclusive and exclusive start + conditions is still a little vague, here's a simple example + illustrating the connection between the two. The set of + rules: + + %s example + %% + + <example>foo do_something(); + + bar something_else(); + + is equivalent to + + %x example + %% + + <example>foo do_something(); + + <INITIAL,example>bar something_else(); + + Without the <INITIAL,example> qualifier, the bar pattern in + the second example wouldn't be active (i.e., couldn't match) + when in start condition example. If we just used <example> + to qualify bar, though, then it would only be active in + example and not in INITIAL, while in the first example it's + active in both, because in the first example the example + startion condition is an inclusive (%s) start condition. + + Also note that the special start-condition specifier <*> + matches every start condition. Thus, the above example + could also have been written; + + %x example + %% + + + + +Version 2.5 Last change: April 1995 18 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + <example>foo do_something(); + + <*>bar something_else(); + + + The default rule (to ECHO any unmatched character) remains + active in start conditions. It is equivalent to: + + <*>.|\n ECHO; + + + BEGIN(0) returns to the original state where only the rules + with no start conditions are active. This state can also be + referred to as the start-condition "INITIAL", so + BEGIN(INITIAL) is equivalent to BEGIN(0). (The parentheses + around the start condition name are not required but are + considered good style.) + + BEGIN actions can also be given as indented code at the + beginning of the rules section. For example, the following + will cause the scanner to enter the "SPECIAL" start condi- + tion whenever yylex() is called and the global variable + enter_special is true: + + int enter_special; + + %x SPECIAL + %% + if ( enter_special ) + BEGIN(SPECIAL); + + <SPECIAL>blahblahblah + ...more rules follow... + + + To illustrate the uses of start conditions, here is a + scanner which provides two different interpretations of a + string like "123.456". By default it will treat it as three + tokens, the integer "123", a dot ('.'), and the integer + "456". But if the string is preceded earlier in the line by + the string "expect-floats" it will treat it as a single + token, the floating-point number 123.456: + + %{ + #include <math.h> + %} + %s expect + + %% + expect-floats BEGIN(expect); + + <expect>[0-9]+"."[0-9]+ { + + + +Version 2.5 Last change: April 1995 19 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + printf( "found a float, = %f\n", + atof( yytext ) ); + } + <expect>\n { + /* that's the end of the line, so + * we need another "expect-number" + * before we'll recognize any more + * numbers + */ + BEGIN(INITIAL); + } + + [0-9]+ { + printf( "found an integer, = %d\n", + atoi( yytext ) ); + } + + "." printf( "found a dot\n" ); + + Here is a scanner which recognizes (and discards) C comments + while maintaining a count of the current input line. + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + This scanner goes to a bit of trouble to match as much text + as possible with each rule. In general, when attempting to + write a high-speed scanner try to match as much possible in + each rule, as it's a big win. + + Note that start-conditions names are really integer values + and can be stored as such. Thus, the above could be + extended in the following fashion: + + %x comment foo + %% + int line_num = 1; + int comment_caller; + + "/*" { + comment_caller = INITIAL; + BEGIN(comment); + } + + + + +Version 2.5 Last change: April 1995 20 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + ... + + <foo>"/*" { + comment_caller = foo; + BEGIN(comment); + } + + <comment>[^*\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(comment_caller); + + Furthermore, you can access the current start condition + using the integer-valued YY_START macro. For example, the + above assignments to comment_caller could instead be written + + comment_caller = YY_START; + + Flex provides YYSTATE as an alias for YY_START (since that + is what's used by AT&T lex). + + Note that start conditions do not have their own name-space; + %s's and %x's declare names in the same fashion as + #define's. + + Finally, here's an example of how to match C-style quoted + strings using exclusive start conditions, including expanded + escape sequences (but not including checking for a string + that's too long): + + %x str + + %% + char string_buf[MAX_STR_CONST]; + char *string_buf_ptr; + + + \" string_buf_ptr = string_buf; BEGIN(str); + + <str>\" { /* saw closing quote - all done */ + BEGIN(INITIAL); + *string_buf_ptr = '\0'; + /* return string constant token type and + * value to parser + */ + } + + <str>\n { + /* error - unterminated string constant */ + /* generate error message */ + } + + + + +Version 2.5 Last change: April 1995 21 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + <str>\\[0-7]{1,3} { + /* octal escape sequence */ + int result; + + (void) sscanf( yytext + 1, "%o", &result ); + + if ( result > 0xff ) + /* error, constant is out-of-bounds */ + + *string_buf_ptr++ = result; + } + + <str>\\[0-9]+ { + /* generate error - bad escape sequence; something + * like '\48' or '\0777777' + */ + } + + <str>\\n *string_buf_ptr++ = '\n'; + <str>\\t *string_buf_ptr++ = '\t'; + <str>\\r *string_buf_ptr++ = '\r'; + <str>\\b *string_buf_ptr++ = '\b'; + <str>\\f *string_buf_ptr++ = '\f'; + + <str>\\(.|\n) *string_buf_ptr++ = yytext[1]; + + <str>[^\\\n\"]+ { + char *yptr = yytext; + + while ( *yptr ) + *string_buf_ptr++ = *yptr++; + } + + + Often, such as in some of the examples above, you wind up + writing a whole bunch of rules all preceded by the same + start condition(s). Flex makes this a little easier and + cleaner by introducing a notion of start condition scope. A + start condition scope is begun with: + + <SCs>{ + + where SCs is a list of one or more start conditions. Inside + the start condition scope, every rule automatically has the + prefix <SCs> applied to it, until a '}' which matches the + initial '{'. So, for example, + + <ESC>{ + "\\n" return '\n'; + "\\r" return '\r'; + "\\f" return '\f'; + "\\0" return '\0'; + + + +Version 2.5 Last change: April 1995 22 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + } + + is equivalent to: + + <ESC>"\\n" return '\n'; + <ESC>"\\r" return '\r'; + <ESC>"\\f" return '\f'; + <ESC>"\\0" return '\0'; + + Start condition scopes may be nested. + + Three routines are available for manipulating stacks of + start conditions: + + void yy_push_state(int new_state) + pushes the current start condition onto the top of the + start condition stack and switches to new_state as + though you had used BEGIN new_state (recall that start + condition names are also integers). + + void yy_pop_state() + pops the top of the stack and switches to it via BEGIN. + + int yy_top_state() + returns the top of the stack without altering the + stack's contents. + + The start condition stack grows dynamically and so has no + built-in size limitation. If memory is exhausted, program + execution aborts. + + To use start condition stacks, your scanner must include a + %option stack directive (see Options below). + +MULTIPLE INPUT BUFFERS + Some scanners (such as those which support "include" files) + require reading from several input streams. As flex + scanners do a large amount of buffering, one cannot control + where the next input will be read from by simply writing a + YY_INPUT which is sensitive to the scanning context. + YY_INPUT is only called when the scanner reaches the end of + its buffer, which may be a long time after scanning a state- + ment such as an "include" which requires switching the input + source. + + To negotiate these sorts of problems, flex provides a + mechanism for creating and switching between multiple input + buffers. An input buffer is created by using: + + YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) + + which takes a FILE pointer and a size and creates a buffer + + + +Version 2.5 Last change: April 1995 23 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + associated with the given file and large enough to hold size + characters (when in doubt, use YY_BUF_SIZE for the size). + It returns a YY_BUFFER_STATE handle, which may then be + passed to other routines (see below). The YY_BUFFER_STATE + type is a pointer to an opaque struct yy_buffer_state struc- + ture, so you may safely initialize YY_BUFFER_STATE variables + to ((YY_BUFFER_STATE) 0) if you wish, and also refer to the + opaque structure in order to correctly declare input buffers + in source files other than that of your scanner. Note that + the FILE pointer in the call to yy_create_buffer is only + used as the value of yyin seen by YY_INPUT; if you redefine + YY_INPUT so it no longer uses yyin, then you can safely pass + a nil FILE pointer to yy_create_buffer. You select a partic- + ular buffer to scan from using: + + void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) + + switches the scanner's input buffer so subsequent tokens + will come from new_buffer. Note that yy_switch_to_buffer() + may be used by yywrap() to set things up for continued scan- + ning, instead of opening a new file and pointing yyin at it. + Note also that switching input sources via either + yy_switch_to_buffer() or yywrap() does not change the start + condition. + + void yy_delete_buffer( YY_BUFFER_STATE buffer ) + + is used to reclaim the storage associated with a buffer. ( + buffer can be nil, in which case the routine does nothing.) + You can also clear the current contents of a buffer using: + + void yy_flush_buffer( YY_BUFFER_STATE buffer ) + + This function discards the buffer's contents, so the next + time the scanner attempts to match a token from the buffer, + it will first fill the buffer anew using YY_INPUT. + + yy_new_buffer() is an alias for yy_create_buffer(), provided + for compatibility with the C++ use of new and delete for + creating and destroying dynamic objects. + + Finally, the YY_CURRENT_BUFFER macro returns a + YY_BUFFER_STATE handle to the current buffer. + + Here is an example of using these features for writing a + scanner which expands include files (the <<EOF>> feature is + discussed below): + + /* the "incl" state is used for picking up the name + * of an include file + */ + %x incl + + + +Version 2.5 Last change: April 1995 24 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + %{ + #define MAX_INCLUDE_DEPTH 10 + YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH]; + int include_stack_ptr = 0; + %} + + %% + include BEGIN(incl); + + [a-z]+ ECHO; + [^a-z\n]*\n? ECHO; + + <incl>[ \t]* /* eat the whitespace */ + <incl>[^ \t\n]+ { /* got the include file name */ + if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) + { + fprintf( stderr, "Includes nested too deeply" ); + exit( 1 ); + } + + include_stack[include_stack_ptr++] = + YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + + if ( ! yyin ) + error( ... ); + + yy_switch_to_buffer( + yy_create_buffer( yyin, YY_BUF_SIZE ) ); + + BEGIN(INITIAL); + } + + <<EOF>> { + if ( --include_stack_ptr < 0 ) + { + yyterminate(); + } + + else + { + yy_delete_buffer( YY_CURRENT_BUFFER ); + yy_switch_to_buffer( + include_stack[include_stack_ptr] ); + } + } + + Three routines are available for setting up input buffers + for scanning in-memory strings instead of files. All of + them create a new input buffer for scanning the string, and + return a corresponding YY_BUFFER_STATE handle (which you + + + +Version 2.5 Last change: April 1995 25 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + should delete with yy_delete_buffer() when done with it). + They also switch to the new buffer using + yy_switch_to_buffer(), so the next call to yylex() will + start scanning the string. + + yy_scan_string(const char *str) + scans a NUL-terminated string. + + yy_scan_bytes(const char *bytes, int len) + scans len bytes (including possibly NUL's) starting at + location bytes. + + Note that both of these functions create and scan a copy of + the string or bytes. (This may be desirable, since yylex() + modifies the contents of the buffer it is scanning.) You + can avoid the copy by using: + + yy_scan_buffer(char *base, yy_size_t size) + which scans in place the buffer starting at base, con- + sisting of size bytes, the last two bytes of which must + be YY_END_OF_BUFFER_CHAR (ASCII NUL). These last two + bytes are not scanned; thus, scanning consists of + base[0] through base[size-2], inclusive. + + If you fail to set up base in this manner (i.e., forget + the final two YY_END_OF_BUFFER_CHAR bytes), then + yy_scan_buffer() returns a nil pointer instead of + creating a new input buffer. + + The type yy_size_t is an integral type to which you can + cast an integer expression reflecting the size of the + buffer. + +END-OF-FILE RULES + The special rule "<<EOF>>" indicates actions which are to be + taken when an end-of-file is encountered and yywrap() + returns non-zero (i.e., indicates no further files to pro- + cess). The action must finish by doing one of four things: + + - assigning yyin to a new input file (in previous ver- + sions of flex, after doing the assignment you had to + call the special action YY_NEW_FILE; this is no longer + necessary); + + - executing a return statement; + + - executing the special yyterminate() action; + + - or, switching to a new buffer using + yy_switch_to_buffer() as shown in the example above. + + + + + +Version 2.5 Last change: April 1995 26 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + <<EOF>> rules may not be used with other patterns; they may + only be qualified with a list of start conditions. If an + unqualified <<EOF>> rule is given, it applies to all start + conditions which do not already have <<EOF>> actions. To + specify an <<EOF>> rule for only the initial start condi- + tion, use + + <INITIAL><<EOF>> + + + These rules are useful for catching things like unclosed + comments. An example: + + %x quote + %% + + ...other rules for dealing with quotes... + + <quote><<EOF>> { + error( "unterminated quote" ); + yyterminate(); + } + <<EOF>> { + if ( *++filelist ) + yyin = fopen( *filelist, "r" ); + else + yyterminate(); + } + + +MISCELLANEOUS MACROS + The macro YY_USER_ACTION can be defined to provide an action + which is always executed prior to the matched rule's action. + For example, it could be #define'd to call a routine to con- + vert yytext to lower-case. When YY_USER_ACTION is invoked, + the variable yy_act gives the number of the matched rule + (rules are numbered starting with 1). Suppose you want to + profile how often each of your rules is matched. The fol- + lowing would do the trick: + + #define YY_USER_ACTION ++ctr[yy_act] + + where ctr is an array to hold the counts for the different + rules. Note that the macro YY_NUM_RULES gives the total + number of rules (including the default rule, even if you use + -s), so a correct declaration for ctr is: + + int ctr[YY_NUM_RULES]; + + + The macro YY_USER_INIT may be defined to provide an action + which is always executed before the first scan (and before + + + +Version 2.5 Last change: April 1995 27 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + the scanner's internal initializations are done). For exam- + ple, it could be used to call a routine to read in a data + table or open a logging file. + + The macro yy_set_interactive(is_interactive) can be used to + control whether the current buffer is considered interac- + tive. An interactive buffer is processed more slowly, but + must be used when the scanner's input source is indeed + interactive to avoid problems due to waiting to fill buffers + (see the discussion of the -I flag below). A non-zero value + in the macro invocation marks the buffer as interactive, a + zero value as non-interactive. Note that use of this macro + overrides %option always-interactive or %option never- + interactive (see Options below). yy_set_interactive() must + be invoked prior to beginning to scan the buffer that is (or + is not) to be considered interactive. + + The macro yy_set_bol(at_bol) can be used to control whether + the current buffer's scanning context for the next token + match is done as though at the beginning of a line. A non- + zero macro argument makes rules anchored with + + The macro YY_AT_BOL() returns true if the next token scanned + from the current buffer will have '^' rules active, false + otherwise. + + In the generated scanner, the actions are all gathered in + one large switch statement and separated using YY_BREAK, + which may be redefined. By default, it is simply a "break", + to separate each rule's action from the following rule's. + Redefining YY_BREAK allows, for example, C++ users to + #define YY_BREAK to do nothing (while being very careful + that every rule ends with a "break" or a "return"!) to avoid + suffering from unreachable statement warnings where because + a rule's action ends with "return", the YY_BREAK is inacces- + sible. + +VALUES AVAILABLE TO THE USER + This section summarizes the various values available to the + user in the rule actions. + + - char *yytext holds the text of the current token. It + may be modified but not lengthened (you cannot append + characters to the end). + + If the special directive %array appears in the first + section of the scanner description, then yytext is + instead declared char yytext[YYLMAX], where YYLMAX is a + macro definition that you can redefine in the first + section if you don't like the default value (generally + 8KB). Using %array results in somewhat slower + scanners, but the value of yytext becomes immune to + + + +Version 2.5 Last change: April 1995 28 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + calls to input() and unput(), which potentially destroy + its value when yytext is a character pointer. The + opposite of %array is %pointer, which is the default. + + You cannot use %array when generating C++ scanner + classes (the -+ flag). + + - int yyleng holds the length of the current token. + + - FILE *yyin is the file which by default flex reads + from. It may be redefined but doing so only makes + sense before scanning begins or after an EOF has been + encountered. Changing it in the midst of scanning will + have unexpected results since flex buffers its input; + use yyrestart() instead. Once scanning terminates + because an end-of-file has been seen, you can assign + yyin at the new input file and then call the scanner + again to continue scanning. + + - void yyrestart( FILE *new_file ) may be called to point + yyin at the new input file. The switch-over to the new + file is immediate (any previously buffered-up input is + lost). Note that calling yyrestart() with yyin as an + argument thus throws away the current input buffer and + continues scanning the same input file. + + - FILE *yyout is the file to which ECHO actions are done. + It can be reassigned by the user. + + - YY_CURRENT_BUFFER returns a YY_BUFFER_STATE handle to + the current buffer. + + - YY_START returns an integer value corresponding to the + current start condition. You can subsequently use this + value with BEGIN to return to that start condition. + +INTERFACING WITH YACC + One of the main uses of flex is as a companion to the yacc + parser-generator. yacc parsers expect to call a routine + named yylex() to find the next input token. The routine is + supposed to return the type of the next token as well as + putting any associated value in the global yylval. To use + flex with yacc, one specifies the -d option to yacc to + instruct it to generate the file y.tab.h containing defini- + tions of all the %tokens appearing in the yacc input. This + file is then included in the flex scanner. For example, if + one of the tokens is "TOK_NUMBER", part of the scanner might + look like: + + %{ + #include "y.tab.h" + %} + + + +Version 2.5 Last change: April 1995 29 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + %% + + [0-9]+ yylval = atoi( yytext ); return TOK_NUMBER; + + +OPTIONS + flex has the following options: + + -b Generate backing-up information to lex.backup. This is + a list of scanner states which require backing up and + the input characters on which they do so. By adding + rules one can remove backing-up states. If all + backing-up states are eliminated and -Cf or -CF is + used, the generated scanner will run faster (see the -p + flag). Only users who wish to squeeze every last cycle + out of their scanners need worry about this option. + (See the section on Performance Considerations below.) + + -c is a do-nothing, deprecated option included for POSIX + compliance. + + -d makes the generated scanner run in debug mode. When- + ever a pattern is recognized and the global + yy_flex_debug is non-zero (which is the default), the + scanner will write to stderr a line of the form: + + --accepting rule at line 53 ("the matched text") + + The line number refers to the location of the rule in + the file defining the scanner (i.e., the file that was + fed to flex). Messages are also generated when the + scanner backs up, accepts the default rule, reaches the + end of its input buffer (or encounters a NUL; at this + point, the two look the same as far as the scanner's + concerned), or reaches an end-of-file. + + -f specifies fast scanner. No table compression is done + and stdio is bypassed. The result is large but fast. + This option is equivalent to -Cfr (see below). + + -h generates a "help" summary of flex's options to stdout + and then exits. -? and --help are synonyms for -h. + + -i instructs flex to generate a case-insensitive scanner. + The case of letters given in the flex input patterns + will be ignored, and tokens in the input will be + matched regardless of case. The matched text given in + yytext will have the preserved case (i.e., it will not + be folded). + + -l turns on maximum compatibility with the original AT&T + lex implementation. Note that this does not mean full + + + +Version 2.5 Last change: April 1995 30 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + compatibility. Use of this option costs a considerable + amount of performance, and it cannot be used with the + -+, -f, -F, -Cf, or -CF options. For details on the + compatibilities it provides, see the section "Incompa- + tibilities With Lex And POSIX" below. This option also + results in the name YY_FLEX_LEX_COMPAT being #define'd + in the generated scanner. + + -n is another do-nothing, deprecated option included only + for POSIX compliance. + + -p generates a performance report to stderr. The report + consists of comments regarding features of the flex + input file which will cause a serious loss of perfor- + mance in the resulting scanner. If you give the flag + twice, you will also get comments regarding features + that lead to minor performance losses. + + Note that the use of REJECT, %option yylineno, and + variable trailing context (see the Deficiencies / Bugs + section below) entails a substantial performance + penalty; use of yymore(), the ^ operator, and the -I + flag entail minor performance penalties. + + -s causes the default rule (that unmatched scanner input + is echoed to stdout) to be suppressed. If the scanner + encounters input that does not match any of its rules, + it aborts with an error. This option is useful for + finding holes in a scanner's rule set. + + -t instructs flex to write the scanner it generates to + standard output instead of lex.yy.c. + + -v specifies that flex should write to stderr a summary of + statistics regarding the scanner it generates. Most of + the statistics are meaningless to the casual flex user, + but the first line identifies the version of flex (same + as reported by -V), and the next line the flags used + when generating the scanner, including those that are + on by default. + + -w suppresses warning messages. + + -B instructs flex to generate a batch scanner, the oppo- + site of interactive scanners generated by -I (see + below). In general, you use -B when you are certain + that your scanner will never be used interactively, and + you want to squeeze a little more performance out of + it. If your goal is instead to squeeze out a lot more + performance, you should be using the -Cf or -CF + options (discussed below), which turn on -B automati- + cally anyway. + + + +Version 2.5 Last change: April 1995 31 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + -F specifies that the fast scanner table representation + should be used (and stdio bypassed). This representa- + tion is about as fast as the full table representation + (-f), and for some sets of patterns will be consider- + ably smaller (and for others, larger). In general, if + the pattern set contains both "keywords" and a catch- + all, "identifier" rule, such as in the set: + + "case" return TOK_CASE; + "switch" return TOK_SWITCH; + ... + "default" return TOK_DEFAULT; + [a-z]+ return TOK_ID; + + then you're better off using the full table representa- + tion. If only the "identifier" rule is present and you + then use a hash table or some such to detect the key- + words, you're better off using -F. + + This option is equivalent to -CFr (see below). It can- + not be used with -+. + + -I instructs flex to generate an interactive scanner. An + interactive scanner is one that only looks ahead to + decide what token has been matched if it absolutely + must. It turns out that always looking one extra char- + acter ahead, even if the scanner has already seen + enough text to disambiguate the current token, is a bit + faster than only looking ahead when necessary. But + scanners that always look ahead give dreadful interac- + tive performance; for example, when a user types a new- + line, it is not recognized as a newline token until + they enter another token, which often means typing in + another whole line. + + Flex scanners default to interactive unless you use the + -Cf or -CF table-compression options (see below). + That's because if you're looking for high-performance + you should be using one of these options, so if you + didn't, flex assumes you'd rather trade off a bit of + run-time performance for intuitive interactive + behavior. Note also that you cannot use -I in conjunc- + tion with -Cf or -CF. Thus, this option is not really + needed; it is on by default for all those cases in + which it is allowed. + + You can force a scanner to not be interactive by using + -B (see above). + + -L instructs flex not to generate #line directives. + Without this option, flex peppers the generated scanner + with #line directives so error messages in the actions + + + +Version 2.5 Last change: April 1995 32 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + will be correctly located with respect to either the + original flex input file (if the errors are due to code + in the input file), or lex.yy.c (if the errors are + flex's fault -- you should report these sorts of errors + to the email address given below). + + -T makes flex run in trace mode. It will generate a lot + of messages to stderr concerning the form of the input + and the resultant non-deterministic and deterministic + finite automata. This option is mostly for use in + maintaining flex. + + -V prints the version number to stdout and exits. --ver- + sion is a synonym for -V. + + -7 instructs flex to generate a 7-bit scanner, i.e., one + which can only recognized 7-bit characters in its + input. The advantage of using -7 is that the scanner's + tables can be up to half the size of those generated + using the -8 option (see below). The disadvantage is + that such scanners often hang or crash if their input + contains an 8-bit character. + + Note, however, that unless you generate your scanner + using the -Cf or -CF table compression options, use of + -7 will save only a small amount of table space, and + make your scanner considerably less portable. Flex's + default behavior is to generate an 8-bit scanner unless + you use the -Cf or -CF, in which case flex defaults to + generating 7-bit scanners unless your site was always + configured to generate 8-bit scanners (as will often be + the case with non-USA sites). You can tell whether + flex generated a 7-bit or an 8-bit scanner by inspect- + ing the flag summary in the -v output as described + above. + + Note that if you use -Cfe or -CFe (those table compres- + sion options, but also using equivalence classes as + discussed see below), flex still defaults to generating + an 8-bit scanner, since usually with these compression + options full 8-bit tables are not much more expensive + than 7-bit tables. + + -8 instructs flex to generate an 8-bit scanner, i.e., one + which can recognize 8-bit characters. This flag is + only needed for scanners generated using -Cf or -CF, as + otherwise flex defaults to generating an 8-bit scanner + anyway. + + See the discussion of -7 above for flex's default + behavior and the tradeoffs between 7-bit and 8-bit + scanners. + + + +Version 2.5 Last change: April 1995 33 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + -+ specifies that you want flex to generate a C++ scanner + class. See the section on Generating C++ Scanners + below for details. + + -C[aefFmr] + controls the degree of table compression and, more gen- + erally, trade-offs between small scanners and fast + scanners. + + -Ca ("align") instructs flex to trade off larger tables + in the generated scanner for faster performance because + the elements of the tables are better aligned for + memory access and computation. On some RISC architec- + tures, fetching and manipulating longwords is more + efficient than with smaller-sized units such as short- + words. This option can double the size of the tables + used by your scanner. + + -Ce directs flex to construct equivalence classes, + i.e., sets of characters which have identical lexical + properties (for example, if the only appearance of + digits in the flex input is in the character class + "[0-9]" then the digits '0', '1', ..., '9' will all be + put in the same equivalence class). Equivalence + classes usually give dramatic reductions in the final + table/object file sizes (typically a factor of 2-5) and + are pretty cheap performance-wise (one array look-up + per character scanned). + + -Cf specifies that the full scanner tables should be + generated - flex should not compress the tables by tak- + ing advantages of similar transition functions for dif- + ferent states. + + -CF specifies that the alternate fast scanner represen- + tation (described above under the -F flag) should be + used. This option cannot be used with -+. + + -Cm directs flex to construct meta-equivalence classes, + which are sets of equivalence classes (or characters, + if equivalence classes are not being used) that are + commonly used together. Meta-equivalence classes are + often a big win when using compressed tables, but they + have a moderate performance impact (one or two "if" + tests and one array look-up per character scanned). + + -Cr causes the generated scanner to bypass use of the + standard I/O library (stdio) for input. Instead of + calling fread() or getc(), the scanner will use the + read() system call, resulting in a performance gain + which varies from system to system, but in general is + probably negligible unless you are also using -Cf or + + + +Version 2.5 Last change: April 1995 34 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + -CF. Using -Cr can cause strange behavior if, for exam- + ple, you read from yyin using stdio prior to calling + the scanner (because the scanner will miss whatever + text your previous reads left in the stdio input + buffer). + + -Cr has no effect if you define YY_INPUT (see The Gen- + erated Scanner above). + + A lone -C specifies that the scanner tables should be + compressed but neither equivalence classes nor meta- + equivalence classes should be used. + + The options -Cf or -CF and -Cm do not make sense + together - there is no opportunity for meta-equivalence + classes if the table is not being compressed. Other- + wise the options may be freely mixed, and are cumula- + tive. + + The default setting is -Cem, which specifies that flex + should generate equivalence classes and meta- + equivalence classes. This setting provides the highest + degree of table compression. You can trade off + faster-executing scanners at the cost of larger tables + with the following generally being true: + + slowest & smallest + -Cem + -Cm + -Ce + -C + -C{f,F}e + -C{f,F} + -C{f,F}a + fastest & largest + + Note that scanners with the smallest tables are usually + generated and compiled the quickest, so during develop- + ment you will usually want to use the default, maximal + compression. + + -Cfe is often a good compromise between speed and size + for production scanners. + + -ooutput + directs flex to write the scanner to the file output + instead of lex.yy.c. If you combine -o with the -t + option, then the scanner is written to stdout but its + #line directives (see the -L option above) refer to the + file output. + + -Pprefix + + + +Version 2.5 Last change: April 1995 35 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + changes the default yy prefix used by flex for all + globally-visible variable and function names to instead + be prefix. For example, -Pfoo changes the name of + yytext to footext. It also changes the name of the + default output file from lex.yy.c to lex.foo.c. Here + are all of the names affected: + + yy_create_buffer + yy_delete_buffer + yy_flex_debug + yy_init_buffer + yy_flush_buffer + yy_load_buffer_state + yy_switch_to_buffer + yyin + yyleng + yylex + yylineno + yyout + yyrestart + yytext + yywrap + + (If you are using a C++ scanner, then only yywrap and + yyFlexLexer are affected.) Within your scanner itself, + you can still refer to the global variables and func- + tions using either version of their name; but exter- + nally, they have the modified name. + + This option lets you easily link together multiple flex + programs into the same executable. Note, though, that + using this option also renames yywrap(), so you now + must either provide your own (appropriately-named) ver- + sion of the routine for your scanner, or use %option + noyywrap, as linking with -lfl no longer provides one + for you by default. + + -Sskeleton_file + overrides the default skeleton file from which flex + constructs its scanners. You'll never need this option + unless you are doing flex maintenance or development. + + flex also provides a mechanism for controlling options + within the scanner specification itself, rather than from + the flex command-line. This is done by including %option + directives in the first section of the scanner specifica- + tion. You can specify multiple options with a single + %option directive, and multiple directives in the first sec- + tion of your flex input file. + + Most options are given simply as names, optionally preceded + by the word "no" (with no intervening whitespace) to negate + + + +Version 2.5 Last change: April 1995 36 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + their meaning. A number are equivalent to flex flags or + their negation: + + 7bit -7 option + 8bit -8 option + align -Ca option + backup -b option + batch -B option + c++ -+ option + + caseful or + case-sensitive opposite of -i (default) + + case-insensitive or + caseless -i option + + debug -d option + default opposite of -s option + ecs -Ce option + fast -F option + full -f option + interactive -I option + lex-compat -l option + meta-ecs -Cm option + perf-report -p option + read -Cr option + stdout -t option + verbose -v option + warn opposite of -w option + (use "%option nowarn" for -w) + + array equivalent to "%array" + pointer equivalent to "%pointer" (default) + + Some %option's provide features otherwise not available: + + always-interactive + instructs flex to generate a scanner which always con- + siders its input "interactive". Normally, on each new + input file the scanner calls isatty() in an attempt to + determine whether the scanner's input source is + interactive and thus should be read a character at a + time. When this option is used, however, then no such + call is made. + + main directs flex to provide a default main() program for + the scanner, which simply calls yylex(). This option + implies noyywrap (see below). + + never-interactive + instructs flex to generate a scanner which never con- + siders its input "interactive" (again, no call made to + + + +Version 2.5 Last change: April 1995 37 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + isatty()). This is the opposite of always-interactive. + + stack + enables the use of start condition stacks (see Start + Conditions above). + + stdinit + if set (i.e., %option stdinit) initializes yyin and + yyout to stdin and stdout, instead of the default of + nil. Some existing lex programs depend on this + behavior, even though it is not compliant with ANSI C, + which does not require stdin and stdout to be compile- + time constant. + + yylineno + directs flex to generate a scanner that maintains the + number of the current line read from its input in the + global variable yylineno. This option is implied by + %option lex-compat. + + yywrap + if unset (i.e., %option noyywrap), makes the scanner + not call yywrap() upon an end-of-file, but simply + assume that there are no more files to scan (until the + user points yyin at a new file and calls yylex() + again). + + flex scans your rule actions to determine whether you use + the REJECT or yymore() features. The reject and yymore + options are available to override its decision as to whether + you use the options, either by setting them (e.g., %option + reject) to indicate the feature is indeed used, or unsetting + them to indicate it actually is not used (e.g., %option + noyymore). + + Three options take string-delimited values, offset with '=': + + %option outfile="ABC" + + is equivalent to -oABC, and + + %option prefix="XYZ" + + is equivalent to -PXYZ. Finally, + + %option yyclass="foo" + + only applies when generating a C++ scanner ( -+ option). It + informs flex that you have derived foo as a subclass of + yyFlexLexer, so flex will place your actions in the member + function foo::yylex() instead of yyFlexLexer::yylex(). It + also generates a yyFlexLexer::yylex() member function that + + + +Version 2.5 Last change: April 1995 38 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + emits a run-time error (by invoking + yyFlexLexer::LexerError()) if called. See Generating C++ + Scanners, below, for additional information. + + A number of options are available for lint purists who want + to suppress the appearance of unneeded routines in the gen- + erated scanner. Each of the following, if unset (e.g., + %option nounput ), results in the corresponding routine not + appearing in the generated scanner: + + input, unput + yy_push_state, yy_pop_state, yy_top_state + yy_scan_buffer, yy_scan_bytes, yy_scan_string + + (though yy_push_state() and friends won't appear anyway + unless you use %option stack). + +PERFORMANCE CONSIDERATIONS + The main design goal of flex is that it generate high- + performance scanners. It has been optimized for dealing + well with large sets of rules. Aside from the effects on + scanner speed of the table compression -C options outlined + above, there are a number of options/actions which degrade + performance. These are, from most expensive to least: + + REJECT + %option yylineno + arbitrary trailing context + + pattern sets that require backing up + %array + %option interactive + %option always-interactive + + '^' beginning-of-line operator + yymore() + + with the first three all being quite expensive and the last + two being quite cheap. Note also that unput() is imple- + mented as a routine call that potentially does quite a bit + of work, while yyless() is a quite-cheap macro; so if just + putting back some excess text you scanned, use yyless(). + + REJECT should be avoided at all costs when performance is + important. It is a particularly expensive option. + + Getting rid of backing up is messy and often may be an enor- + mous amount of work for a complicated scanner. In princi- + pal, one begins by using the -b flag to generate a + lex.backup file. For example, on the input + + %% + + + +Version 2.5 Last change: April 1995 39 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + the file looks like: + + State #6 is non-accepting - + associated rule line numbers: + 2 3 + out-transitions: [ o ] + jam-transitions: EOF [ \001-n p-\177 ] + + State #8 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ a ] + jam-transitions: EOF [ \001-` b-\177 ] + + State #9 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ r ] + jam-transitions: EOF [ \001-q s-\177 ] + + Compressed tables always back up. + + The first few lines tell us that there's a scanner state in + which it can make a transition on an 'o' but not on any + other character, and that in that state the currently + scanned text does not match any rule. The state occurs when + trying to match the rules found at lines 2 and 3 in the + input file. If the scanner is in that state and then reads + something other than an 'o', it will have to back up to find + a rule which is matched. With a bit of headscratching one + can see that this must be the state it's in when it has seen + "fo". When this has happened, if anything other than + another 'o' is seen, the scanner will have to back up to + simply match the 'f' (by the default rule). + + The comment regarding State #8 indicates there's a problem + when "foob" has been scanned. Indeed, on any character + other than an 'a', the scanner will have to back up to + accept "foo". Similarly, the comment for State #9 concerns + when "fooba" has been scanned and an 'r' does not follow. + + The final comment reminds us that there's no point going to + all the trouble of removing backing up from the rules unless + we're using -Cf or -CF, since there's no performance gain + doing so with compressed scanners. + + The way to remove the backing up is to add "error" rules: + + %% + + + +Version 2.5 Last change: April 1995 40 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + fooba | + foob | + fo { + /* false alarm, not really a keyword */ + return TOK_ID; + } + + + Eliminating backing up among a list of keywords can also be + done using a "catch-all" rule: + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + [a-z]+ return TOK_ID; + + This is usually the best solution when appropriate. + + Backing up messages tend to cascade. With a complicated set + of rules it's not uncommon to get hundreds of messages. If + one can decipher them, though, it often only takes a dozen + or so rules to eliminate the backing up (though it's easy to + make a mistake and have an error rule accidentally match a + valid token. A possible future flex feature will be to + automatically add rules to eliminate backing up). + + It's important to keep in mind that you gain the benefits of + eliminating backing up only if you eliminate every instance + of backing up. Leaving just one means you gain nothing. + + Variable trailing context (where both the leading and trail- + ing parts do not have a fixed length) entails almost the + same performance loss as REJECT (i.e., substantial). So + when possible a rule like: + + %% + mouse|rat/(cat|dog) run(); + + is better written: + + %% + mouse/cat|dog run(); + rat/cat|dog run(); + + or as + + %% + mouse|rat/cat run(); + + + +Version 2.5 Last change: April 1995 41 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + mouse|rat/dog run(); + + Note that here the special '|' action does not provide any + savings, and can even make things worse (see Deficiencies / + Bugs below). + + Another area where the user can increase a scanner's perfor- + mance (and one that's easier to implement) arises from the + fact that the longer the tokens matched, the faster the + scanner will run. This is because with long tokens the pro- + cessing of most input characters takes place in the (short) + inner scanning loop, and does not often have to go through + the additional work of setting up the scanning environment + (e.g., yytext) for the action. Recall the scanner for C + comments: + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* + <comment>"*"+[^*/\n]* + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + This could be sped up by writing it as: + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* + <comment>[^*\n]*\n ++line_num; + <comment>"*"+[^*/\n]* + <comment>"*"+[^*/\n]*\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + Now instead of each newline requiring the processing of + another action, recognizing the newlines is "distributed" + over the other rules to keep the matched text as long as + possible. Note that adding rules does not slow down the + scanner! The speed of the scanner is independent of the + number of rules or (modulo the considerations given at the + beginning of this section) how complicated the rules are + with regard to operators such as '*' and '|'. + + A final example in speeding up a scanner: suppose you want + to scan through a file containing identifiers and keywords, + + + +Version 2.5 Last change: April 1995 42 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + one per line and with no other extraneous characters, and + recognize all the keywords. A natural first approach is: + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + .|\n /* it's not a keyword */ + + To eliminate the back-tracking, introduce a catch-all rule: + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + [a-z]+ | + .|\n /* it's not a keyword */ + + Now, if it's guaranteed that there's exactly one word per + line, then we can reduce the total number of matches by a + half by merging in the recognition of newlines with that of + the other tokens: + + %% + asm\n | + auto\n | + break\n | + ... etc ... + volatile\n | + while\n /* it's a keyword */ + + [a-z]+\n | + .|\n /* it's not a keyword */ + + One has to be careful here, as we have now reintroduced + backing up into the scanner. In particular, while we know + that there will never be any characters in the input stream + other than letters or newlines, flex can't figure this out, + and it will plan for possibly needing to back up when it has + scanned a token like "auto" and then the next character is + something other than a newline or a letter. Previously it + would then just match the "auto" rule and be done, but now + it has no "auto" rule, only a "auto\n" rule. To eliminate + the possibility of backing up, we could either duplicate all + + + +Version 2.5 Last change: April 1995 43 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + rules but without final newlines, or, since we never expect + to encounter such an input and therefore don't how it's + classified, we can introduce one more catch-all rule, this + one which doesn't include a newline: + + %% + asm\n | + auto\n | + break\n | + ... etc ... + volatile\n | + while\n /* it's a keyword */ + + [a-z]+\n | + [a-z]+ | + .|\n /* it's not a keyword */ + + Compiled with -Cf, this is about as fast as one can get a + flex scanner to go for this particular problem. + + A final note: flex is slow when matching NUL's, particularly + when a token contains multiple NUL's. It's best to write + rules which match short amounts of text if it's anticipated + that the text will often include NUL's. + + Another final note regarding performance: as mentioned above + in the section How the Input is Matched, dynamically resiz- + ing yytext to accommodate huge tokens is a slow process + because it presently requires that the (huge) token be res- + canned from the beginning. Thus if performance is vital, + you should attempt to match "large" quantities of text but + not "huge" quantities, where the cutoff between the two is + at about 8K characters/token. + +GENERATING C++ SCANNERS + flex provides two different ways to generate scanners for + use with C++. The first way is to simply compile a scanner + generated by flex using a C++ compiler instead of a C com- + piler. You should not encounter any compilations errors + (please report any you find to the email address given in + the Author section below). You can then use C++ code in + your rule actions instead of C code. Note that the default + input source for your scanner remains yyin, and default + echoing is still done to yyout. Both of these remain FILE * + variables and not C++ streams. + + You can also use flex to generate a C++ scanner class, using + the -+ option (or, equivalently, %option c++), which is + automatically specified if the name of the flex executable + ends in a '+', such as flex++. When using this option, flex + defaults to generating the scanner to the file lex.yy.cc + instead of lex.yy.c. The generated scanner includes the + + + +Version 2.5 Last change: April 1995 44 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + header file FlexLexer.h, which defines the interface to two + C++ classes. + + The first class, FlexLexer, provides an abstract base class + defining the general scanner class interface. It provides + the following member functions: + + const char* YYText() + returns the text of the most recently matched token, + the equivalent of yytext. + + int YYLeng() + returns the length of the most recently matched token, + the equivalent of yyleng. + + int lineno() const + returns the current input line number (see %option + yylineno), or 1 if %option yylineno was not used. + + void set_debug( int flag ) + sets the debugging flag for the scanner, equivalent to + assigning to yy_flex_debug (see the Options section + above). Note that you must build the scanner using + %option debug to include debugging information in it. + + int debug() const + returns the current setting of the debugging flag. + + Also provided are member functions equivalent to + yy_switch_to_buffer(), yy_create_buffer() (though the first + argument is an istream* object pointer and not a FILE*), + yy_flush_buffer(), yy_delete_buffer(), and yyrestart() + (again, the first argument is a istream* object pointer). + + The second class defined in FlexLexer.h is yyFlexLexer, + which is derived from FlexLexer. It defines the following + additional member functions: + + yyFlexLexer( istream* arg_yyin = 0, ostream* arg_yyout = 0 ) + constructs a yyFlexLexer object using the given streams + for input and output. If not specified, the streams + default to cin and cout, respectively. + + virtual int yylex() + performs the same role is yylex() does for ordinary + flex scanners: it scans the input stream, consuming + tokens, until a rule's action returns a value. If you + derive a subclass S from yyFlexLexer and want to access + the member functions and variables of S inside yylex(), + then you need to use %option yyclass="S" to inform flex + that you will be using that subclass instead of yyFlex- + Lexer. In this case, rather than generating + + + +Version 2.5 Last change: April 1995 45 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + yyFlexLexer::yylex(), flex generates S::yylex() (and + also generates a dummy yyFlexLexer::yylex() that calls + yyFlexLexer::LexerError() if called). + + virtual void switch_streams(istream* new_in = 0, + ostream* new_out = 0) reassigns yyin to new_in (if + non-nil) and yyout to new_out (ditto), deleting the + previous input buffer if yyin is reassigned. + + int yylex( istream* new_in, ostream* new_out = 0 ) + first switches the input streams via switch_streams( + new_in, new_out ) and then returns the value of + yylex(). + + In addition, yyFlexLexer defines the following protected + virtual functions which you can redefine in derived classes + to tailor the scanner: + + virtual int LexerInput( char* buf, int max_size ) + reads up to max_size characters into buf and returns + the number of characters read. To indicate end-of- + input, return 0 characters. Note that "interactive" + scanners (see the -B and -I flags) define the macro + YY_INTERACTIVE. If you redefine LexerInput() and need + to take different actions depending on whether or not + the scanner might be scanning an interactive input + source, you can test for the presence of this name via + #ifdef. + + virtual void LexerOutput( const char* buf, int size ) + writes out size characters from the buffer buf, which, + while NUL-terminated, may also contain "internal" NUL's + if the scanner's rules can match text with NUL's in + them. + + virtual void LexerError( const char* msg ) + reports a fatal error message. The default version of + this function writes the message to the stream cerr and + exits. + + Note that a yyFlexLexer object contains its entire scanning + state. Thus you can use such objects to create reentrant + scanners. You can instantiate multiple instances of the + same yyFlexLexer class, and you can also combine multiple + C++ scanner classes together in the same program using the + -P option discussed above. + + Finally, note that the %array feature is not available to + C++ scanner classes; you must use %pointer (the default). + + Here is an example of a simple C++ scanner: + + + + +Version 2.5 Last change: April 1995 46 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + // An example of using the flex C++ scanner class. + + %{ + int mylineno = 0; + %} + + string \"[^\n"]+\" + + ws [ \t]+ + + alpha [A-Za-z] + dig [0-9] + name ({alpha}|{dig}|\$)({alpha}|{dig}|[_.\-/$])* + num1 [-+]?{dig}+\.?([eE][-+]?{dig}+)? + num2 [-+]?{dig}*\.{dig}+([eE][-+]?{dig}+)? + number {num1}|{num2} + + %% + + {ws} /* skip blanks and tabs */ + + "/*" { + int c; + + while((c = yyinput()) != 0) + { + if(c == '\n') + ++mylineno; + + else if(c == '*') + { + if((c = yyinput()) == '/') + break; + else + unput(c); + } + } + } + + {number} cout << "number " << YYText() << '\n'; + + \n mylineno++; + + {name} cout << "name " << YYText() << '\n'; + + {string} cout << "string " << YYText() << '\n'; + + %% + + int main( int /* argc */, char** /* argv */ ) + { + FlexLexer* lexer = new yyFlexLexer; + + + +Version 2.5 Last change: April 1995 47 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + while(lexer->yylex() != 0) + ; + return 0; + } + If you want to create multiple (different) lexer classes, + you use the -P flag (or the prefix= option) to rename each + yyFlexLexer to some other xxFlexLexer. You then can include + <FlexLexer.h> in your other sources once per lexer class, + first renaming yyFlexLexer as follows: + + #undef yyFlexLexer + #define yyFlexLexer xxFlexLexer + #include <FlexLexer.h> + + #undef yyFlexLexer + #define yyFlexLexer zzFlexLexer + #include <FlexLexer.h> + + if, for example, you used %option prefix="xx" for one of + your scanners and %option prefix="zz" for the other. + + IMPORTANT: the present form of the scanning class is experi- + mental and may change considerably between major releases. + +INCOMPATIBILITIES WITH LEX AND POSIX + flex is a rewrite of the AT&T Unix lex tool (the two imple- + mentations do not share any code, though), with some exten- + sions and incompatibilities, both of which are of concern to + those who wish to write scanners acceptable to either imple- + mentation. Flex is fully compliant with the POSIX lex + specification, except that when using %pointer (the + default), a call to unput() destroys the contents of yytext, + which is counter to the POSIX specification. + + In this section we discuss all of the known areas of incom- + patibility between flex, AT&T lex, and the POSIX specifica- + tion. + + flex's -l option turns on maximum compatibility with the + original AT&T lex implementation, at the cost of a major + loss in the generated scanner's performance. We note below + which incompatibilities can be overcome using the -l option. + + flex is fully compatible with lex with the following excep- + tions: + + - The undocumented lex scanner internal variable yylineno + is not supported unless -l or %option yylineno is used. + + yylineno should be maintained on a per-buffer basis, + rather than a per-scanner (single global variable) + basis. + + + +Version 2.5 Last change: April 1995 48 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + yylineno is not part of the POSIX specification. + + - The input() routine is not redefinable, though it may + be called to read characters following whatever has + been matched by a rule. If input() encounters an end- + of-file the normal yywrap() processing is done. A + ``real'' end-of-file is returned by input() as EOF. + + Input is instead controlled by defining the YY_INPUT + macro. + + The flex restriction that input() cannot be redefined + is in accordance with the POSIX specification, which + simply does not specify any way of controlling the + scanner's input other than by making an initial assign- + ment to yyin. + + - The unput() routine is not redefinable. This restric- + tion is in accordance with POSIX. + + - flex scanners are not as reentrant as lex scanners. In + particular, if you have an interactive scanner and an + interrupt handler which long-jumps out of the scanner, + and the scanner is subsequently called again, you may + get the following message: + + fatal flex scanner internal error--end of buffer missed + + To reenter the scanner, first use + + yyrestart( yyin ); + + Note that this call will throw away any buffered input; + usually this isn't a problem with an interactive + scanner. + + Also note that flex C++ scanner classes are reentrant, + so if using C++ is an option for you, you should use + them instead. See "Generating C++ Scanners" above for + details. + + - output() is not supported. Output from the ECHO macro + is done to the file-pointer yyout (default stdout). + + output() is not part of the POSIX specification. + + - lex does not support exclusive start conditions (%x), + though they are in the POSIX specification. + + - When definitions are expanded, flex encloses them in + parentheses. With lex, the following: + + + + +Version 2.5 Last change: April 1995 49 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + NAME [A-Z][A-Z0-9]* + %% + foo{NAME}? printf( "Found it\n" ); + %% + + will not match the string "foo" because when the macro + is expanded the rule is equivalent to "foo[A-Z][A-Z0- + 9]*?" and the precedence is such that the '?' is asso- + ciated with "[A-Z0-9]*". With flex, the rule will be + expanded to "foo([A-Z][A-Z0-9]*)?" and so the string + "foo" will match. + + Note that if the definition begins with ^ or ends with + $ then it is not expanded with parentheses, to allow + these operators to appear in definitions without losing + their special meanings. But the <s>, /, and <<EOF>> + operators cannot be used in a flex definition. + + Using -l results in the lex behavior of no parentheses + around the definition. + + The POSIX specification is that the definition be + enclosed in parentheses. + + - Some implementations of lex allow a rule's action to + begin on a separate line, if the rule's pattern has + trailing whitespace: + + %% + foo|bar<space here> + { foobar_action(); } + + flex does not support this feature. + + - The lex %r (generate a Ratfor scanner) option is not + supported. It is not part of the POSIX specification. + + - After a call to unput(), yytext is undefined until the + next token is matched, unless the scanner was built + using %array. This is not the case with lex or the + POSIX specification. The -l option does away with this + incompatibility. + + - The precedence of the {} (numeric range) operator is + different. lex interprets "abc{1,3}" as "match one, + two, or three occurrences of 'abc'", whereas flex + interprets it as "match 'ab' followed by one, two, or + three occurrences of 'c'". The latter is in agreement + with the POSIX specification. + + - The precedence of the ^ operator is different. lex + interprets "^foo|bar" as "match either 'foo' at the + + + +Version 2.5 Last change: April 1995 50 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + beginning of a line, or 'bar' anywhere", whereas flex + interprets it as "match either 'foo' or 'bar' if they + come at the beginning of a line". The latter is in + agreement with the POSIX specification. + + - The special table-size declarations such as %a sup- + ported by lex are not required by flex scanners; flex + ignores them. + + - The name FLEX_SCANNER is #define'd so scanners may be + written for use with either flex or lex. Scanners also + include YY_FLEX_MAJOR_VERSION and YY_FLEX_MINOR_VERSION + indicating which version of flex generated the scanner + (for example, for the 2.5 release, these defines would + be 2 and 5 respectively). + + The following flex features are not included in lex or the + POSIX specification: + + C++ scanners + %option + start condition scopes + start condition stacks + interactive/non-interactive scanners + yy_scan_string() and friends + yyterminate() + yy_set_interactive() + yy_set_bol() + YY_AT_BOL() + <<EOF>> + <*> + YY_DECL + YY_START + YY_USER_ACTION + YY_USER_INIT + #line directives + %{}'s around actions + multiple actions on a line + + plus almost all of the flex flags. The last feature in the + list refers to the fact that with flex you can put multiple + actions on the same line, separated with semi-colons, while + with lex, the following + + foo handle_foo(); ++num_foos_seen; + + is (rather surprisingly) truncated to + + foo handle_foo(); + + flex does not truncate the action. Actions that are not + enclosed in braces are simply terminated at the end of the + + + +Version 2.5 Last change: April 1995 51 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + line. + +DIAGNOSTICS + warning, rule cannot be matched indicates that the given + rule cannot be matched because it follows other rules that + will always match the same text as it. For example, in the + following "foo" cannot be matched because it comes after an + identifier "catch-all" rule: + + [a-z]+ got_identifier(); + foo got_foo(); + + Using REJECT in a scanner suppresses this warning. + + warning, -s option given but default rule can be matched + means that it is possible (perhaps only in a particular + start condition) that the default rule (match any single + character) is the only one that will match a particular + input. Since -s was given, presumably this is not intended. + + reject_used_but_not_detected undefined or + yymore_used_but_not_detected undefined - These errors can + occur at compile time. They indicate that the scanner uses + REJECT or yymore() but that flex failed to notice the fact, + meaning that flex scanned the first two sections looking for + occurrences of these actions and failed to find any, but + somehow you snuck some in (via a #include file, for exam- + ple). Use %option reject or %option yymore to indicate to + flex that you really do use these features. + + flex scanner jammed - a scanner compiled with -s has encoun- + tered an input string which wasn't matched by any of its + rules. This error can also occur due to internal problems. + + token too large, exceeds YYLMAX - your scanner uses %array + and one of its rules matched a string longer than the YYLMAX + constant (8K bytes by default). You can increase the value + by #define'ing YYLMAX in the definitions section of your + flex input. + + scanner requires -8 flag to use the character 'x' - Your + scanner specification includes recognizing the 8-bit charac- + ter 'x' and you did not specify the -8 flag, and your + scanner defaulted to 7-bit because you used the -Cf or -CF + table compression options. See the discussion of the -7 + flag for details. + + flex scanner push-back overflow - you used unput() to push + back so much text that the scanner's buffer could not hold + both the pushed-back text and the current token in yytext. + Ideally the scanner should dynamically resize the buffer in + this case, but at present it does not. + + + +Version 2.5 Last change: April 1995 52 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + input buffer overflow, can't enlarge buffer because scanner + uses REJECT - the scanner was working on matching an + extremely large token and needed to expand the input buffer. + This doesn't work with scanners that use REJECT. + + fatal flex scanner internal error--end of buffer missed - + This can occur in an scanner which is reentered after a + long-jump has jumped out (or over) the scanner's activation + frame. Before reentering the scanner, use: + + yyrestart( yyin ); + + or, as noted above, switch to using the C++ scanner class. + + too many start conditions in <> you listed more start condi- + tions in a <> construct than exist (so you must have listed + at least one of them twice). + +FILES + -lfl library with which scanners must be linked. + + lex.yy.c + generated scanner (called lexyy.c on some systems). + + lex.yy.cc + generated C++ scanner class, when using -+. + + <FlexLexer.h> + header file defining the C++ scanner base class, Flex- + Lexer, and its derived class, yyFlexLexer. + + flex.skl + skeleton scanner. This file is only used when building + flex, not when flex executes. + + lex.backup + backing-up information for -b flag (called lex.bck on + some systems). + +DEFICIENCIES / BUGS + Some trailing context patterns cannot be properly matched + and generate warning messages ("dangerous trailing con- + text"). These are patterns where the ending of the first + part of the rule matches the beginning of the second part, + such as "zx*/xy*", where the 'x*' matches the 'x' at the + beginning of the trailing context. (Note that the POSIX + draft states that the text matched by such patterns is unde- + fined.) + + For some trailing context rules, parts which are actually + fixed-length are not recognized as such, leading to the + abovementioned performance loss. In particular, parts using + + + +Version 2.5 Last change: April 1995 53 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + '|' or {n} (such as "foo{3}") are always considered + variable-length. + + Combining trailing context with the special '|' action can + result in fixed trailing context being turned into the more + expensive variable trailing context. For example, in the + following: + + %% + abc | + xyz/def + + + Use of unput() invalidates yytext and yyleng, unless the + %array directive or the -l option has been used. + + Pattern-matching of NUL's is substantially slower than + matching other characters. + + Dynamic resizing of the input buffer is slow, as it entails + rescanning all the text matched so far by the current (gen- + erally huge) token. + + Due to both buffering of input and read-ahead, you cannot + intermix calls to <stdio.h> routines, such as, for example, + getchar(), with flex rules and expect it to work. Call + input() instead. + + The total table entries listed by the -v flag excludes the + number of table entries needed to determine what rule has + been matched. The number of entries is equal to the number + of DFA states if the scanner does not use REJECT, and some- + what greater than the number of states if it does. + + REJECT cannot be used with the -f or -F options. + + The flex internal algorithms need documentation. + +SEE ALSO + lex(1), yacc(1), sed(1), awk(1). + + John Levine, Tony Mason, and Doug Brown, Lex & Yacc, + O'Reilly and Associates. Be sure to get the 2nd edition. + + M. E. Lesk and E. Schmidt, LEX - Lexical Analyzer Generator + + Alfred Aho, Ravi Sethi and Jeffrey Ullman, Compilers: Prin- + ciples, Techniques and Tools, Addison-Wesley (1986). + Describes the pattern-matching techniques used by flex + (deterministic finite automata). + + + + + +Version 2.5 Last change: April 1995 54 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + +AUTHOR + Vern Paxson, with the help of many ideas and much inspira- + tion from Van Jacobson. Original version by Jef Poskanzer. + The fast table representation is a partial implementation of + a design done by Van Jacobson. The implementation was done + by Kevin Gong and Vern Paxson. + + Thanks to the many flex beta-testers, feedbackers, and con- + tributors, especially Francois Pinard, Casey Leedom, Robert + Abramovitz, Stan Adermann, Terry Allen, David Barker- + Plummer, John Basrai, Neal Becker, Nelson H.F. Beebe, + benson@odi.com, Karl Berry, Peter A. Bigot, Simon Blanchard, + Keith Bostic, Frederic Brehm, Ian Brockbank, Kin Cho, Nick + Christopher, Brian Clapper, J.T. Conklin, Jason Coughlin, + Bill Cox, Nick Cropper, Dave Curtis, Scott David Daniels, + Chris G. Demetriou, Theo Deraadt, Mike Donahue, Chuck + Doucette, Tom Epperly, Leo Eskin, Chris Faylor, Chris + Flatters, Jon Forrest, Jeffrey Friedl, Joe Gayda, Kaveh R. + Ghazi, Wolfgang Glunz, Eric Goldman, Christopher M. Gould, + Ulrich Grepel, Peer Griebel, Jan Hajic, Charles Hemphill, + NORO Hideo, Jarkko Hietaniemi, Scott Hofmann, Jeff Honig, + Dana Hudes, Eric Hughes, John Interrante, Ceriel Jacobs, + Michal Jaegermann, Sakari Jalovaara, Jeffrey R. Jones, Henry + Juengst, Klaus Kaempf, Jonathan I. Kamens, Terrence O Kane, + Amir Katz, ken@ken.hilco.com, Kevin B. Kenny, Steve Kirsch, + Winfried Koenig, Marq Kole, Ronald Lamprecht, Greg Lee, + Rohan Lenard, Craig Leres, John Levine, Steve Liddle, David + Loffredo, Mike Long, Mohamed el Lozy, Brian Madsen, Malte, + Joe Marshall, Bengt Martensson, Chris Metcalf, Luke Mewburn, + Jim Meyering, R. Alexander Milowski, Erik Naggum, G.T. + Nicol, Landon Noll, James Nordby, Marc Nozell, Richard + Ohnemus, Karsten Pahnke, Sven Panne, Roland Pesch, Walter + Pelissero, Gaumond Pierre, Esmond Pitt, Jef Poskanzer, Joe + Rahmeh, Jarmo Raiha, Frederic Raimbault, Pat Rankin, Rick + Richardson, Kevin Rodgers, Kai Uwe Rommel, Jim Roskind, + Alberto Santini, Andreas Scherer, Darrell Schiebel, Raf + Schietekat, Doug Schmidt, Philippe Schnoebelen, Andreas + Schwab, Larry Schwimmer, Alex Siegel, Eckehard Stolz, Jan- + Erik Strvmquist, Mike Stump, Paul Stuart, Dave Tallman, Ian + Lance Taylor, Chris Thewalt, Richard M. Timoney, Jodi Tsai, + Paul Tuinenga, Gary Weik, Frank Whaley, Gerhard Wilhelms, + Kent Williams, Ken Yap, Ron Zellar, Nathan Zelle, David + Zuhn, and those whose names have slipped my marginal mail- + archiving skills but whose contributions are appreciated all + the same. + + Thanks to Keith Bostic, Jon Forrest, Noah Friedman, John + Gilmore, Craig Leres, John Levine, Bob Mulcahy, G.T. Nicol, + Francois Pinard, Rich Salz, and Richard Stallman for help + with various distribution headaches. + + + + + +Version 2.5 Last change: April 1995 55 + + + + + + +FLEX(1) USER COMMANDS FLEX(1) + + + + Thanks to Esmond Pitt and Earle Horton for 8-bit character + support; to Benson Margulies and Fred Burke for C++ support; + to Kent Williams and Tom Epperly for C++ class support; to + Ove Ewerlid for support of NUL's; and to Eric Hughes for + support of multiple buffers. + + This work was primarily done when I was with the Real Time + Systems Group at the Lawrence Berkeley Laboratory in Berke- + ley, CA. Many thanks to all there for the support I + received. + + Send comments to vern@ee.lbl.gov. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Version 2.5 Last change: April 1995 56 + + + diff --git a/MISC/parse.c b/MISC/parse.c new file mode 100644 index 0000000..fea9b91 --- /dev/null +++ b/MISC/parse.c @@ -0,0 +1,1452 @@ +#ifndef lint +static char yysccsid[] = "@(#)yaccpar 1.9 (Berkeley) 02/21/93"; +#endif +#define YYBYACC 1 +#define YYMAJOR 1 +#define YYMINOR 9 +#define yyclearin (yychar=(-1)) +#define yyerrok (yyerrflag=0) +#define YYRECOVERING (yyerrflag!=0) +#define YYPREFIX "yy" +#line 10 "./parse.y" +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Vern Paxson. + * + * The United States Government has rights in this work pursuant + * to contract no. DE-AC03-76SF00098 between the United States + * Department of Energy and the University of California. + * + * Redistribution and use in source and binary forms with or without + * modification are permitted provided that: (1) source distributions retain + * this entire copyright notice and comment, and (2) distributions including + * binaries display the following acknowledgement: ``This product includes + * software developed by the University of California, Berkeley and its + * contributors'' in the documentation or other materials provided with the + * distribution and in all advertising materials mentioning features or use + * of this software. Neither the name of the University nor the names of + * its contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* $Header: /home/daffy/u0/vern/flex/RCS/parse.y,v 2.28 95/04/21 11:51:51 vern Exp $ */ + + +/* Some versions of bison are broken in that they use alloca() but don't + * declare it properly. The following is the patented (just kidding!) + * #ifdef chud to fix the problem, courtesy of Francois Pinard. + */ +#ifdef YYBISON +/* AIX requires this to be the first thing in the file. What a piece. */ +# ifdef _AIX + #pragma alloca +# endif +#endif + +#include "flexdef.h" + +/* The remainder of the alloca() cruft has to come after including flexdef.h, + * so HAVE_ALLOCA_H is (possibly) defined. + */ +#ifdef YYBISON +# ifdef __GNUC__ +# ifndef alloca +# define alloca __builtin_alloca +# endif +# else +# if HAVE_ALLOCA_H +# include <alloca.h> +# else +# ifdef __hpux +void *alloca (); +# else +# ifdef __TURBOC__ +# include <malloc.h> +# else +char *alloca (); +# endif +# endif +# endif +# endif +#endif + +/* Bletch, ^^^^ that was ugly! */ + + +int pat, scnum, eps, headcnt, trailcnt, anyccl, lastchar, i, rulelen; +int trlcontxt, xcluflg, currccl, cclsorted, varlength, variable_trail_rule; + +int *scon_stk; +int scon_stk_ptr; + +static int madeany = false; /* whether we've made the '.' character class */ +int previous_continued_action; /* whether the previous rule's action was '|' */ + +/* Expand a POSIX character class expression. */ +#define CCL_EXPR(func) \ + { \ + int c; \ + for ( c = 0; c < csize; ++c ) \ + if ( isascii(c) && func(c) ) \ + ccladd( currccl, c ); \ + } + +/* While POSIX defines isblank(), it's not ANSI C. */ +#define IS_BLANK(c) ((c) == ' ' || (c) == '\t') + +/* On some over-ambitious machines, such as DEC Alpha's, the default + * token type is "long" instead of "int"; this leads to problems with + * declaring yylval in flexdef.h. But so far, all the yacc's I've seen + * wrap their definitions of YYSTYPE with "#ifndef YYSTYPE"'s, so the + * following should ensure that the default token type is "int". + */ +#define YYSTYPE int + +#line 112 "y.tab.c" +#define CHAR 257 +#define NUMBER 258 +#define SECTEND 259 +#define SCDECL 260 +#define XSCDECL 261 +#define NAME 262 +#define PREVCCL 263 +#define EOF_OP 264 +#define OPTION_OP 265 +#define OPT_OUTFILE 266 +#define OPT_PREFIX 267 +#define OPT_YYCLASS 268 +#define CCE_ALNUM 269 +#define CCE_ALPHA 270 +#define CCE_BLANK 271 +#define CCE_CNTRL 272 +#define CCE_DIGIT 273 +#define CCE_GRAPH 274 +#define CCE_LOWER 275 +#define CCE_PRINT 276 +#define CCE_PUNCT 277 +#define CCE_SPACE 278 +#define CCE_UPPER 279 +#define CCE_XDIGIT 280 +#define YYERRCODE 256 +short yylhs[] = { -1, + 0, 1, 2, 2, 2, 2, 3, 6, 6, 7, + 7, 7, 8, 9, 9, 10, 10, 10, 4, 4, + 4, 5, 12, 12, 12, 12, 14, 11, 11, 11, + 15, 15, 15, 16, 13, 13, 13, 13, 18, 18, + 17, 19, 19, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 21, 21, 23, 23, 23, + 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 22, 22, +}; +short yylen[] = { 2, + 5, 0, 3, 2, 0, 1, 1, 1, 1, 2, + 1, 1, 2, 2, 0, 3, 3, 3, 5, 5, + 0, 0, 2, 1, 1, 1, 0, 4, 3, 0, + 3, 1, 1, 1, 2, 3, 2, 1, 3, 1, + 2, 2, 1, 2, 2, 2, 6, 5, 4, 1, + 1, 1, 3, 3, 1, 3, 4, 4, 2, 2, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 2, 0, +}; +short yydefred[] = { 2, + 0, 0, 6, 0, 7, 8, 9, 15, 21, 0, + 4, 0, 0, 12, 11, 0, 0, 0, 0, 14, + 0, 1, 0, 10, 0, 0, 0, 0, 0, 21, + 0, 16, 17, 18, 29, 33, 34, 0, 32, 0, + 26, 55, 52, 25, 0, 50, 75, 0, 0, 0, + 24, 0, 0, 0, 0, 51, 28, 0, 20, 23, + 0, 0, 61, 0, 19, 0, 37, 0, 41, 0, + 0, 44, 45, 46, 31, 74, 53, 54, 0, 0, + 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 56, 60, 36, 0, 0, 57, 0, 49, + 0, 58, 0, 48, 47, +}; +short yydgoto[] = { 1, + 2, 4, 9, 13, 22, 10, 16, 11, 12, 20, + 23, 50, 51, 29, 38, 39, 52, 53, 54, 55, + 56, 61, 64, 94, +}; +short yysindex[] = { 0, + 0, -235, 0, -191, 0, 0, 0, 0, 0, -207, + 0, -215, -18, 0, 0, -202, 4, 26, 32, 0, + 41, 0, -35, 0, -168, -166, -165, 38, -180, 0, + -30, 0, 0, 0, 0, 0, 0, -16, 0, -40, + 0, 0, 0, 0, -2, 0, 0, -2, 8, 93, + 0, -2, -25, -2, 15, 0, 0, -153, 0, 0, + -27, -26, 0, -88, 0, -23, 0, -2, 0, 15, + -150, 0, 0, 0, 0, 0, 0, 0, -3, 65, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, -2, -21, 0, -145, 0, + -116, 0, -12, 0, 0, +}; +short yyrindex[] = { 0, + 0, -188, 0, 0, 0, 0, 0, 0, 0, 0, + 0, -154, 1, 0, 0, -140, 0, 0, 0, 0, + -176, 0, -28, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, -32, + 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, + 0, 0, 106, 7, -10, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 108, 0, 0, 0, -7, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, + 0, 0, 0, 0, 0, +}; +short yygindex[] = { 0, + 0, 0, 0, 92, 100, 0, 0, 0, 0, 0, + 0, 0, 81, 0, 0, 69, 0, 27, 60, -29, + 0, 0, 66, 0, +}; +#define YYTABLESIZE 326 +short yytable[] = { 43, + 22, 30, 42, 47, 93, 22, 77, 30, 104, 48, + 67, 22, 95, 30, 78, 46, 40, 22, 39, 21, + 3, 69, 101, 43, 70, 43, 42, 58, 42, 43, + 43, 47, 42, 42, 30, 43, 43, 48, 42, 42, + 30, 21, 40, 46, 39, 57, 30, 40, 14, 39, + 17, 18, 19, 40, 15, 39, 72, 73, 30, 24, + 49, 30, 22, 45, 25, 22, 70, 5, 6, 7, + 5, 5, 5, 8, 62, 36, 5, 74, 66, 27, + 43, 37, 28, 42, 59, 27, 26, 30, 49, 98, + 30, 30, 27, 32, 30, 33, 34, 68, 68, 35, + 68, 63, 65, 100, 13, 13, 13, 97, 37, 99, + 13, 102, 105, 43, 61, 38, 42, 35, 3, 3, + 3, 40, 31, 30, 3, 60, 75, 96, 79, 0, + 40, 0, 39, 0, 0, 0, 0, 71, 59, 0, + 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 80, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + 91, 92, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 30, 30, 41, 42, 22, 22, 76, + 30, 30, 43, 44, 22, 22, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 43, 0, 0, 42, + 0, 0, 43, 80, 42, 42, 30, 30, 0, 0, + 43, 0, 0, 30, 30, 81, 82, 83, 84, 85, + 86, 87, 88, 89, 90, 91, 92, 0, 61, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, + 61, 61, 59, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 59, 59, 59, 59, 59, 59, + 59, 59, 59, 59, 59, 59, +}; +short yycheck[] = { 10, + 0, 34, 10, 34, 93, 34, 34, 40, 125, 40, + 36, 40, 36, 46, 41, 46, 10, 46, 10, 60, + 256, 47, 44, 34, 54, 36, 34, 44, 36, 40, + 41, 34, 40, 41, 34, 46, 47, 40, 46, 47, + 40, 60, 36, 46, 36, 62, 46, 41, 256, 41, + 266, 267, 268, 47, 262, 47, 42, 43, 91, 262, + 91, 94, 91, 94, 61, 94, 96, 259, 260, 261, + 259, 260, 261, 265, 48, 256, 265, 63, 52, 256, + 91, 262, 42, 91, 125, 262, 61, 123, 91, 93, + 123, 91, 61, 262, 94, 262, 262, 124, 124, 62, + 124, 94, 10, 125, 259, 260, 261, 258, 262, 45, + 265, 257, 125, 124, 93, 10, 124, 10, 259, 260, + 261, 30, 23, 123, 265, 45, 58, 68, 63, -1, + 124, -1, 124, -1, -1, -1, -1, 123, 93, -1, + -1, 258, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, 257, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 256, 257, 256, 257, 256, 257, 257, + 263, 264, 263, 264, 263, 264, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 257, -1, -1, 257, + -1, -1, 263, 257, 257, 263, 256, 257, -1, -1, + 263, -1, -1, 263, 264, 269, 270, 271, 272, 273, + 274, 275, 276, 277, 278, 279, 280, -1, 257, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, + 279, 280, 257, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 269, 270, 271, 272, 273, 274, + 275, 276, 277, 278, 279, 280, +}; +#define YYFINAL 1 +#ifndef YYDEBUG +#define YYDEBUG 0 +#endif +#define YYMAXTOKEN 280 +#if YYDEBUG +char *yyname[] = { +"end-of-file",0,0,0,0,0,0,0,0,0,"'\\n'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,"'\"'",0,"'$'",0,0,0,"'('","')'","'*'","'+'","','","'-'","'.'","'/'",0,0, +0,0,0,0,0,0,0,0,0,0,"'<'","'='","'>'","'?'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,"'['",0,"']'","'^'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,"'{'","'|'","'}'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, +0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"CHAR","NUMBER","SECTEND", +"SCDECL","XSCDECL","NAME","PREVCCL","EOF_OP","OPTION_OP","OPT_OUTFILE", +"OPT_PREFIX","OPT_YYCLASS","CCE_ALNUM","CCE_ALPHA","CCE_BLANK","CCE_CNTRL", +"CCE_DIGIT","CCE_GRAPH","CCE_LOWER","CCE_PRINT","CCE_PUNCT","CCE_SPACE", +"CCE_UPPER","CCE_XDIGIT", +}; +char *yyrule[] = { +"$accept : goal", +"goal : initlex sect1 sect1end sect2 initforrule", +"initlex :", +"sect1 : sect1 startconddecl namelist1", +"sect1 : sect1 options", +"sect1 :", +"sect1 : error", +"sect1end : SECTEND", +"startconddecl : SCDECL", +"startconddecl : XSCDECL", +"namelist1 : namelist1 NAME", +"namelist1 : NAME", +"namelist1 : error", +"options : OPTION_OP optionlist", +"optionlist : optionlist option", +"optionlist :", +"option : OPT_OUTFILE '=' NAME", +"option : OPT_PREFIX '=' NAME", +"option : OPT_YYCLASS '=' NAME", +"sect2 : sect2 scon initforrule flexrule '\\n'", +"sect2 : sect2 scon '{' sect2 '}'", +"sect2 :", +"initforrule :", +"flexrule : '^' rule", +"flexrule : rule", +"flexrule : EOF_OP", +"flexrule : error", +"scon_stk_ptr :", +"scon : '<' scon_stk_ptr namelist2 '>'", +"scon : '<' '*' '>'", +"scon :", +"namelist2 : namelist2 ',' sconname", +"namelist2 : sconname", +"namelist2 : error", +"sconname : NAME", +"rule : re2 re", +"rule : re2 re '$'", +"rule : re '$'", +"rule : re", +"re : re '|' series", +"re : series", +"re2 : re '/'", +"series : series singleton", +"series : singleton", +"singleton : singleton '*'", +"singleton : singleton '+'", +"singleton : singleton '?'", +"singleton : singleton '{' NUMBER ',' NUMBER '}'", +"singleton : singleton '{' NUMBER ',' '}'", +"singleton : singleton '{' NUMBER '}'", +"singleton : '.'", +"singleton : fullccl", +"singleton : PREVCCL", +"singleton : '\"' string '\"'", +"singleton : '(' re ')'", +"singleton : CHAR", +"fullccl : '[' ccl ']'", +"fullccl : '[' '^' ccl ']'", +"ccl : ccl CHAR '-' CHAR", +"ccl : ccl CHAR", +"ccl : ccl ccl_expr", +"ccl :", +"ccl_expr : CCE_ALNUM", +"ccl_expr : CCE_ALPHA", +"ccl_expr : CCE_BLANK", +"ccl_expr : CCE_CNTRL", +"ccl_expr : CCE_DIGIT", +"ccl_expr : CCE_GRAPH", +"ccl_expr : CCE_LOWER", +"ccl_expr : CCE_PRINT", +"ccl_expr : CCE_PUNCT", +"ccl_expr : CCE_SPACE", +"ccl_expr : CCE_UPPER", +"ccl_expr : CCE_XDIGIT", +"string : string CHAR", +"string :", +}; +#endif +#ifndef YYSTYPE +typedef int YYSTYPE; +#endif +#ifdef YYSTACKSIZE +#undef YYMAXDEPTH +#define YYMAXDEPTH YYSTACKSIZE +#else +#ifdef YYMAXDEPTH +#define YYSTACKSIZE YYMAXDEPTH +#else +#define YYSTACKSIZE 500 +#define YYMAXDEPTH 500 +#endif +#endif +int yydebug; +int yynerrs; +int yyerrflag; +int yychar; +short *yyssp; +YYSTYPE *yyvsp; +YYSTYPE yyval; +YYSTYPE yylval; +short yyss[YYSTACKSIZE]; +YYSTYPE yyvs[YYSTACKSIZE]; +#define yystacksize YYSTACKSIZE +#line 776 "./parse.y" + + +/* build_eof_action - build the "<<EOF>>" action for the active start + * conditions + */ + +void build_eof_action() + { + register int i; + char action_text[MAXLINE]; + + for ( i = 1; i <= scon_stk_ptr; ++i ) + { + if ( sceof[scon_stk[i]] ) + format_pinpoint_message( + "multiple <<EOF>> rules for start condition %s", + scname[scon_stk[i]] ); + + else + { + sceof[scon_stk[i]] = true; + sprintf( action_text, "case YY_STATE_EOF(%s):\n", + scname[scon_stk[i]] ); + add_action( action_text ); + } + } + + line_directive_out( (FILE *) 0, 1 ); + + /* This isn't a normal rule after all - don't count it as + * such, so we don't have any holes in the rule numbering + * (which make generating "rule can never match" warnings + * more difficult. + */ + --num_rules; + ++num_eof_rules; + } + + +/* format_synerr - write out formatted syntax error */ + +void format_synerr( msg, arg ) +char msg[], arg[]; + { + char errmsg[MAXLINE]; + + (void) sprintf( errmsg, msg, arg ); + synerr( errmsg ); + } + + +/* synerr - report a syntax error */ + +void synerr( str ) +char str[]; + { + syntaxerror = true; + pinpoint_message( str ); + } + + +/* format_warn - write out formatted warning */ + +void format_warn( msg, arg ) +char msg[], arg[]; + { + char warn_msg[MAXLINE]; + + (void) sprintf( warn_msg, msg, arg ); + warn( warn_msg ); + } + + +/* warn - report a warning, unless -w was given */ + +void warn( str ) +char str[]; + { + line_warning( str, linenum ); + } + +/* format_pinpoint_message - write out a message formatted with one string, + * pinpointing its location + */ + +void format_pinpoint_message( msg, arg ) +char msg[], arg[]; + { + char errmsg[MAXLINE]; + + (void) sprintf( errmsg, msg, arg ); + pinpoint_message( errmsg ); + } + + +/* pinpoint_message - write out a message, pinpointing its location */ + +void pinpoint_message( str ) +char str[]; + { + line_pinpoint( str, linenum ); + } + + +/* line_warning - report a warning at a given line, unless -w was given */ + +void line_warning( str, line ) +char str[]; +int line; + { + char warning[MAXLINE]; + + if ( ! nowarn ) + { + sprintf( warning, "warning, %s", str ); + line_pinpoint( warning, line ); + } + } + + +/* line_pinpoint - write out a message, pinpointing it at the given line */ + +void line_pinpoint( str, line ) +char str[]; +int line; + { + fprintf( stderr, "\"%s\", line %d: %s\n", infilename, line, str ); + } + + +/* yyerror - eat up an error message from the parser; + * currently, messages are ignore + */ + +void yyerror( msg ) +char msg[]; + { + } +#line 541 "y.tab.c" +#define YYABORT goto yyabort +#define YYREJECT goto yyabort +#define YYACCEPT goto yyaccept +#define YYERROR goto yyerrlab +int +yyparse() +{ + register int yym, yyn, yystate; +#if YYDEBUG + register char *yys; + extern char *getenv(); + + if (yys = getenv("YYDEBUG")) + { + yyn = *yys; + if (yyn >= '0' && yyn <= '9') + yydebug = yyn - '0'; + } +#endif + + yynerrs = 0; + yyerrflag = 0; + yychar = (-1); + + yyssp = yyss; + yyvsp = yyvs; + *yyssp = yystate = 0; + +yyloop: + if (yyn = yydefred[yystate]) goto yyreduce; + if (yychar < 0) + { + if ((yychar = yylex()) < 0) yychar = 0; +#if YYDEBUG + if (yydebug) + { + yys = 0; + if (yychar <= YYMAXTOKEN) yys = yyname[yychar]; + if (!yys) yys = "illegal-symbol"; + printf("%sdebug: state %d, reading %d (%s)\n", + YYPREFIX, yystate, yychar, yys); + } +#endif + } + if ((yyn = yysindex[yystate]) && (yyn += yychar) >= 0 && + yyn <= YYTABLESIZE && yycheck[yyn] == yychar) + { +#if YYDEBUG + if (yydebug) + printf("%sdebug: state %d, shifting to state %d\n", + YYPREFIX, yystate, yytable[yyn]); +#endif + if (yyssp >= yyss + yystacksize - 1) + { + goto yyoverflow; + } + *++yyssp = yystate = yytable[yyn]; + *++yyvsp = yylval; + yychar = (-1); + if (yyerrflag > 0) --yyerrflag; + goto yyloop; + } + if ((yyn = yyrindex[yystate]) && (yyn += yychar) >= 0 && + yyn <= YYTABLESIZE && yycheck[yyn] == yychar) + { + yyn = yytable[yyn]; + goto yyreduce; + } + if (yyerrflag) goto yyinrecovery; +#ifdef lint + goto yynewerror; +#endif +yynewerror: + yyerror("syntax error"); +#ifdef lint + goto yyerrlab; +#endif +yyerrlab: + ++yynerrs; +yyinrecovery: + if (yyerrflag < 3) + { + yyerrflag = 3; + for (;;) + { + if ((yyn = yysindex[*yyssp]) && (yyn += YYERRCODE) >= 0 && + yyn <= YYTABLESIZE && yycheck[yyn] == YYERRCODE) + { +#if YYDEBUG + if (yydebug) + printf("%sdebug: state %d, error recovery shifting\ + to state %d\n", YYPREFIX, *yyssp, yytable[yyn]); +#endif + if (yyssp >= yyss + yystacksize - 1) + { + goto yyoverflow; + } + *++yyssp = yystate = yytable[yyn]; + *++yyvsp = yylval; + goto yyloop; + } + else + { +#if YYDEBUG + if (yydebug) + printf("%sdebug: error recovery discarding state %d\n", + YYPREFIX, *yyssp); +#endif + if (yyssp <= yyss) goto yyabort; + --yyssp; + --yyvsp; + } + } + } + else + { + if (yychar == 0) goto yyabort; +#if YYDEBUG + if (yydebug) + { + yys = 0; + if (yychar <= YYMAXTOKEN) yys = yyname[yychar]; + if (!yys) yys = "illegal-symbol"; + printf("%sdebug: state %d, error recovery discards token %d (%s)\n", + YYPREFIX, yystate, yychar, yys); + } +#endif + yychar = (-1); + goto yyloop; + } +yyreduce: +#if YYDEBUG + if (yydebug) + printf("%sdebug: state %d, reducing by rule %d (%s)\n", + YYPREFIX, yystate, yyn, yyrule[yyn]); +#endif + yym = yylen[yyn]; + yyval = yyvsp[1-yym]; + switch (yyn) + { +case 1: +#line 113 "./parse.y" +{ /* add default rule */ + int def_rule; + + pat = cclinit(); + cclnegate( pat ); + + def_rule = mkstate( -pat ); + + /* Remember the number of the default rule so we + * don't generate "can't match" warnings for it. + */ + default_rule = num_rules; + + finish_rule( def_rule, false, 0, 0 ); + + for ( i = 1; i <= lastsc; ++i ) + scset[i] = mkbranch( scset[i], def_rule ); + + if ( spprdflt ) + add_action( + "YY_FATAL_ERROR( \"flex scanner jammed\" )" ); + else + add_action( "ECHO" ); + + add_action( ";\n\tYY_BREAK\n" ); + } +break; +case 2: +#line 142 "./parse.y" +{ /* initialize for processing rules */ + + /* Create default DFA start condition. */ + scinstal( "INITIAL", false ); + } +break; +case 6: +#line 153 "./parse.y" +{ synerr( "unknown error processing section 1" ); } +break; +case 7: +#line 157 "./parse.y" +{ + check_options(); + scon_stk = allocate_integer_array( lastsc + 1 ); + scon_stk_ptr = 0; + } +break; +case 8: +#line 165 "./parse.y" +{ xcluflg = false; } +break; +case 9: +#line 168 "./parse.y" +{ xcluflg = true; } +break; +case 10: +#line 172 "./parse.y" +{ scinstal( nmstr, xcluflg ); } +break; +case 11: +#line 175 "./parse.y" +{ scinstal( nmstr, xcluflg ); } +break; +case 12: +#line 178 "./parse.y" +{ synerr( "bad start condition list" ); } +break; +case 16: +#line 189 "./parse.y" +{ + outfilename = copy_string( nmstr ); + did_outfilename = 1; + } +break; +case 17: +#line 194 "./parse.y" +{ prefix = copy_string( nmstr ); } +break; +case 18: +#line 196 "./parse.y" +{ yyclass = copy_string( nmstr ); } +break; +case 19: +#line 200 "./parse.y" +{ scon_stk_ptr = yyvsp[-3]; } +break; +case 20: +#line 202 "./parse.y" +{ scon_stk_ptr = yyvsp[-3]; } +break; +case 22: +#line 207 "./parse.y" +{ + /* Initialize for a parse of one rule. */ + trlcontxt = variable_trail_rule = varlength = false; + trailcnt = headcnt = rulelen = 0; + current_state_type = STATE_NORMAL; + previous_continued_action = continued_action; + in_rule = true; + + new_rule(); + } +break; +case 23: +#line 220 "./parse.y" +{ + pat = yyvsp[0]; + finish_rule( pat, variable_trail_rule, + headcnt, trailcnt ); + + if ( scon_stk_ptr > 0 ) + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + scbol[scon_stk[i]] = + mkbranch( scbol[scon_stk[i]], + pat ); + } + + else + { + /* Add to all non-exclusive start conditions, + * including the default (0) start condition. + */ + + for ( i = 1; i <= lastsc; ++i ) + if ( ! scxclu[i] ) + scbol[i] = mkbranch( scbol[i], + pat ); + } + + if ( ! bol_needed ) + { + bol_needed = true; + + if ( performance_report > 1 ) + pinpoint_message( + "'^' operator results in sub-optimal performance" ); + } + } +break; +case 24: +#line 256 "./parse.y" +{ + pat = yyvsp[0]; + finish_rule( pat, variable_trail_rule, + headcnt, trailcnt ); + + if ( scon_stk_ptr > 0 ) + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + scset[scon_stk[i]] = + mkbranch( scset[scon_stk[i]], + pat ); + } + + else + { + for ( i = 1; i <= lastsc; ++i ) + if ( ! scxclu[i] ) + scset[i] = + mkbranch( scset[i], + pat ); + } + } +break; +case 25: +#line 280 "./parse.y" +{ + if ( scon_stk_ptr > 0 ) + build_eof_action(); + + else + { + /* This EOF applies to all start conditions + * which don't already have EOF actions. + */ + for ( i = 1; i <= lastsc; ++i ) + if ( ! sceof[i] ) + scon_stk[++scon_stk_ptr] = i; + + if ( scon_stk_ptr == 0 ) + warn( + "all start conditions already have <<EOF>> rules" ); + + else + build_eof_action(); + } + } +break; +case 26: +#line 303 "./parse.y" +{ synerr( "unrecognized rule" ); } +break; +case 27: +#line 307 "./parse.y" +{ yyval = scon_stk_ptr; } +break; +case 28: +#line 311 "./parse.y" +{ yyval = yyvsp[-2]; } +break; +case 29: +#line 314 "./parse.y" +{ + yyval = scon_stk_ptr; + + for ( i = 1; i <= lastsc; ++i ) + { + int j; + + for ( j = 1; j <= scon_stk_ptr; ++j ) + if ( scon_stk[j] == i ) + break; + + if ( j > scon_stk_ptr ) + scon_stk[++scon_stk_ptr] = i; + } + } +break; +case 30: +#line 331 "./parse.y" +{ yyval = scon_stk_ptr; } +break; +case 33: +#line 339 "./parse.y" +{ synerr( "bad start condition list" ); } +break; +case 34: +#line 343 "./parse.y" +{ + if ( (scnum = sclookup( nmstr )) == 0 ) + format_pinpoint_message( + "undeclared start condition %s", + nmstr ); + else + { + for ( i = 1; i <= scon_stk_ptr; ++i ) + if ( scon_stk[i] == scnum ) + { + format_warn( + "<%s> specified twice", + scname[scnum] ); + break; + } + + if ( i > scon_stk_ptr ) + scon_stk[++scon_stk_ptr] = scnum; + } + } +break; +case 35: +#line 366 "./parse.y" +{ + if ( transchar[lastst[yyvsp[0]]] != SYM_EPSILON ) + /* Provide final transition \now/ so it + * will be marked as a trailing context + * state. + */ + yyvsp[0] = link_machines( yyvsp[0], + mkstate( SYM_EPSILON ) ); + + mark_beginning_as_normal( yyvsp[0] ); + current_state_type = STATE_NORMAL; + + if ( previous_continued_action ) + { + /* We need to treat this as variable trailing + * context so that the backup does not happen + * in the action but before the action switch + * statement. If the backup happens in the + * action, then the rules "falling into" this + * one's action will *also* do the backup, + * erroneously. + */ + if ( ! varlength || headcnt != 0 ) + warn( + "trailing context made variable due to preceding '|' action" ); + + /* Mark as variable. */ + varlength = true; + headcnt = 0; + } + + if ( lex_compat || (varlength && headcnt == 0) ) + { /* variable trailing context rule */ + /* Mark the first part of the rule as the + * accepting "head" part of a trailing + * context rule. + * + * By the way, we didn't do this at the + * beginning of this production because back + * then current_state_type was set up for a + * trail rule, and add_accept() can create + * a new state ... + */ + add_accept( yyvsp[-1], + num_rules | YY_TRAILING_HEAD_MASK ); + variable_trail_rule = true; + } + + else + trailcnt = rulelen; + + yyval = link_machines( yyvsp[-1], yyvsp[0] ); + } +break; +case 36: +#line 421 "./parse.y" +{ synerr( "trailing context used twice" ); } +break; +case 37: +#line 424 "./parse.y" +{ + headcnt = 0; + trailcnt = 1; + rulelen = 1; + varlength = false; + + current_state_type = STATE_TRAILING_CONTEXT; + + if ( trlcontxt ) + { + synerr( "trailing context used twice" ); + yyval = mkstate( SYM_EPSILON ); + } + + else if ( previous_continued_action ) + { + /* See the comment in the rule for "re2 re" + * above. + */ + warn( + "trailing context made variable due to preceding '|' action" ); + + varlength = true; + } + + if ( lex_compat || varlength ) + { + /* Again, see the comment in the rule for + * "re2 re" above. + */ + add_accept( yyvsp[-1], + num_rules | YY_TRAILING_HEAD_MASK ); + variable_trail_rule = true; + } + + trlcontxt = true; + + eps = mkstate( SYM_EPSILON ); + yyval = link_machines( yyvsp[-1], + link_machines( eps, mkstate( '\n' ) ) ); + } +break; +case 38: +#line 467 "./parse.y" +{ + yyval = yyvsp[0]; + + if ( trlcontxt ) + { + if ( lex_compat || (varlength && headcnt == 0) ) + /* Both head and trail are + * variable-length. + */ + variable_trail_rule = true; + else + trailcnt = rulelen; + } + } +break; +case 39: +#line 485 "./parse.y" +{ + varlength = true; + yyval = mkor( yyvsp[-2], yyvsp[0] ); + } +break; +case 40: +#line 491 "./parse.y" +{ yyval = yyvsp[0]; } +break; +case 41: +#line 496 "./parse.y" +{ + /* This rule is written separately so the + * reduction will occur before the trailing + * series is parsed. + */ + + if ( trlcontxt ) + synerr( "trailing context used twice" ); + else + trlcontxt = true; + + if ( varlength ) + /* We hope the trailing context is + * fixed-length. + */ + varlength = false; + else + headcnt = rulelen; + + rulelen = 0; + + current_state_type = STATE_TRAILING_CONTEXT; + yyval = yyvsp[-1]; + } +break; +case 42: +#line 523 "./parse.y" +{ + /* This is where concatenation of adjacent patterns + * gets done. + */ + yyval = link_machines( yyvsp[-1], yyvsp[0] ); + } +break; +case 43: +#line 531 "./parse.y" +{ yyval = yyvsp[0]; } +break; +case 44: +#line 535 "./parse.y" +{ + varlength = true; + + yyval = mkclos( yyvsp[-1] ); + } +break; +case 45: +#line 542 "./parse.y" +{ + varlength = true; + yyval = mkposcl( yyvsp[-1] ); + } +break; +case 46: +#line 548 "./parse.y" +{ + varlength = true; + yyval = mkopt( yyvsp[-1] ); + } +break; +case 47: +#line 554 "./parse.y" +{ + varlength = true; + + if ( yyvsp[-3] > yyvsp[-1] || yyvsp[-3] < 0 ) + { + synerr( "bad iteration values" ); + yyval = yyvsp[-5]; + } + else + { + if ( yyvsp[-3] == 0 ) + { + if ( yyvsp[-1] <= 0 ) + { + synerr( + "bad iteration values" ); + yyval = yyvsp[-5]; + } + else + yyval = mkopt( + mkrep( yyvsp[-5], 1, yyvsp[-1] ) ); + } + else + yyval = mkrep( yyvsp[-5], yyvsp[-3], yyvsp[-1] ); + } + } +break; +case 48: +#line 582 "./parse.y" +{ + varlength = true; + + if ( yyvsp[-2] <= 0 ) + { + synerr( "iteration value must be positive" ); + yyval = yyvsp[-4]; + } + + else + yyval = mkrep( yyvsp[-4], yyvsp[-2], INFINITY ); + } +break; +case 49: +#line 596 "./parse.y" +{ + /* The singleton could be something like "(foo)", + * in which case we have no idea what its length + * is, so we punt here. + */ + varlength = true; + + if ( yyvsp[-1] <= 0 ) + { + synerr( "iteration value must be positive" ); + yyval = yyvsp[-3]; + } + + else + yyval = link_machines( yyvsp[-3], + copysingl( yyvsp[-3], yyvsp[-1] - 1 ) ); + } +break; +case 50: +#line 615 "./parse.y" +{ + if ( ! madeany ) + { + /* Create the '.' character class. */ + anyccl = cclinit(); + ccladd( anyccl, '\n' ); + cclnegate( anyccl ); + + if ( useecs ) + mkeccl( ccltbl + cclmap[anyccl], + ccllen[anyccl], nextecm, + ecgroup, csize, csize ); + + madeany = true; + } + + ++rulelen; + + yyval = mkstate( -anyccl ); + } +break; +case 51: +#line 637 "./parse.y" +{ + if ( ! cclsorted ) + /* Sort characters for fast searching. We + * use a shell sort since this list could + * be large. + */ + cshell( ccltbl + cclmap[yyvsp[0]], ccllen[yyvsp[0]], true ); + + if ( useecs ) + mkeccl( ccltbl + cclmap[yyvsp[0]], ccllen[yyvsp[0]], + nextecm, ecgroup, csize, csize ); + + ++rulelen; + + yyval = mkstate( -yyvsp[0] ); + } +break; +case 52: +#line 655 "./parse.y" +{ + ++rulelen; + + yyval = mkstate( -yyvsp[0] ); + } +break; +case 53: +#line 662 "./parse.y" +{ yyval = yyvsp[-1]; } +break; +case 54: +#line 665 "./parse.y" +{ yyval = yyvsp[-1]; } +break; +case 55: +#line 668 "./parse.y" +{ + ++rulelen; + + if ( caseins && yyvsp[0] >= 'A' && yyvsp[0] <= 'Z' ) + yyvsp[0] = clower( yyvsp[0] ); + + yyval = mkstate( yyvsp[0] ); + } +break; +case 56: +#line 679 "./parse.y" +{ yyval = yyvsp[-1]; } +break; +case 57: +#line 682 "./parse.y" +{ + cclnegate( yyvsp[-1] ); + yyval = yyvsp[-1]; + } +break; +case 58: +#line 689 "./parse.y" +{ + if ( caseins ) + { + if ( yyvsp[-2] >= 'A' && yyvsp[-2] <= 'Z' ) + yyvsp[-2] = clower( yyvsp[-2] ); + if ( yyvsp[0] >= 'A' && yyvsp[0] <= 'Z' ) + yyvsp[0] = clower( yyvsp[0] ); + } + + if ( yyvsp[-2] > yyvsp[0] ) + synerr( "negative range in character class" ); + + else + { + for ( i = yyvsp[-2]; i <= yyvsp[0]; ++i ) + ccladd( yyvsp[-3], i ); + + /* Keep track if this ccl is staying in + * alphabetical order. + */ + cclsorted = cclsorted && (yyvsp[-2] > lastchar); + lastchar = yyvsp[0]; + } + + yyval = yyvsp[-3]; + } +break; +case 59: +#line 717 "./parse.y" +{ + if ( caseins && yyvsp[0] >= 'A' && yyvsp[0] <= 'Z' ) + yyvsp[0] = clower( yyvsp[0] ); + + ccladd( yyvsp[-1], yyvsp[0] ); + cclsorted = cclsorted && (yyvsp[0] > lastchar); + lastchar = yyvsp[0]; + yyval = yyvsp[-1]; + } +break; +case 60: +#line 728 "./parse.y" +{ + /* Too hard to properly maintain cclsorted. */ + cclsorted = false; + yyval = yyvsp[-1]; + } +break; +case 61: +#line 735 "./parse.y" +{ + cclsorted = true; + lastchar = 0; + currccl = yyval = cclinit(); + } +break; +case 62: +#line 742 "./parse.y" +{ CCL_EXPR(isalnum) } +break; +case 63: +#line 743 "./parse.y" +{ CCL_EXPR(isalpha) } +break; +case 64: +#line 744 "./parse.y" +{ CCL_EXPR(IS_BLANK) } +break; +case 65: +#line 745 "./parse.y" +{ CCL_EXPR(iscntrl) } +break; +case 66: +#line 746 "./parse.y" +{ CCL_EXPR(isdigit) } +break; +case 67: +#line 747 "./parse.y" +{ CCL_EXPR(isgraph) } +break; +case 68: +#line 748 "./parse.y" +{ CCL_EXPR(islower) } +break; +case 69: +#line 749 "./parse.y" +{ CCL_EXPR(isprint) } +break; +case 70: +#line 750 "./parse.y" +{ CCL_EXPR(ispunct) } +break; +case 71: +#line 751 "./parse.y" +{ CCL_EXPR(isspace) } +break; +case 72: +#line 752 "./parse.y" +{ + if ( caseins ) + CCL_EXPR(islower) + else + CCL_EXPR(isupper) + } +break; +case 73: +#line 758 "./parse.y" +{ CCL_EXPR(isxdigit) } +break; +case 74: +#line 762 "./parse.y" +{ + if ( caseins && yyvsp[0] >= 'A' && yyvsp[0] <= 'Z' ) + yyvsp[0] = clower( yyvsp[0] ); + + ++rulelen; + + yyval = link_machines( yyvsp[-1], mkstate( yyvsp[0] ) ); + } +break; +case 75: +#line 772 "./parse.y" +{ yyval = mkstate( SYM_EPSILON ); } +break; +#line 1397 "y.tab.c" + } + yyssp -= yym; + yystate = *yyssp; + yyvsp -= yym; + yym = yylhs[yyn]; + if (yystate == 0 && yym == 0) + { +#if YYDEBUG + if (yydebug) + printf("%sdebug: after reduction, shifting from state 0 to\ + state %d\n", YYPREFIX, YYFINAL); +#endif + yystate = YYFINAL; + *++yyssp = YYFINAL; + *++yyvsp = yyval; + if (yychar < 0) + { + if ((yychar = yylex()) < 0) yychar = 0; +#if YYDEBUG + if (yydebug) + { + yys = 0; + if (yychar <= YYMAXTOKEN) yys = yyname[yychar]; + if (!yys) yys = "illegal-symbol"; + printf("%sdebug: state %d, reading %d (%s)\n", + YYPREFIX, YYFINAL, yychar, yys); + } +#endif + } + if (yychar == 0) goto yyaccept; + goto yyloop; + } + if ((yyn = yygindex[yym]) && (yyn += yystate) >= 0 && + yyn <= YYTABLESIZE && yycheck[yyn] == yystate) + yystate = yytable[yyn]; + else + yystate = yydgoto[yym]; +#if YYDEBUG + if (yydebug) + printf("%sdebug: after reduction, shifting from state %d \ +to state %d\n", YYPREFIX, *yyssp, yystate); +#endif + if (yyssp >= yyss + yystacksize - 1) + { + goto yyoverflow; + } + *++yyssp = yystate; + *++yyvsp = yyval; + goto yyloop; +yyoverflow: + yyerror("yacc stack overflow"); +yyabort: + return (1); +yyaccept: + return (0); +} diff --git a/MISC/parse.h b/MISC/parse.h new file mode 100644 index 0000000..10febed --- /dev/null +++ b/MISC/parse.h @@ -0,0 +1,24 @@ +#define CHAR 257 +#define NUMBER 258 +#define SECTEND 259 +#define SCDECL 260 +#define XSCDECL 261 +#define NAME 262 +#define PREVCCL 263 +#define EOF_OP 264 +#define OPTION_OP 265 +#define OPT_OUTFILE 266 +#define OPT_PREFIX 267 +#define OPT_YYCLASS 268 +#define CCE_ALNUM 269 +#define CCE_ALPHA 270 +#define CCE_BLANK 271 +#define CCE_CNTRL 272 +#define CCE_DIGIT 273 +#define CCE_GRAPH 274 +#define CCE_LOWER 275 +#define CCE_PRINT 276 +#define CCE_PUNCT 277 +#define CCE_SPACE 278 +#define CCE_UPPER 279 +#define CCE_XDIGIT 280 diff --git a/MISC/testxxLexer.l b/MISC/testxxLexer.l new file mode 100644 index 0000000..9421541 --- /dev/null +++ b/MISC/testxxLexer.l @@ -0,0 +1,58 @@ + // An example of using the flex C++ scanner class. + +%option C++ noyywrap + +%{ +int mylineno = 0; +%} + +string \"[^\n"]+\" + +ws [ \t]+ + +alpha [A-Za-z] +dig [0-9] +name ({alpha}|{dig}|\$)({alpha}|{dig}|\_|\.|\-|\/|\$)* +num1 [-+]?{dig}+\.?([eE][-+]?{dig}+)? +num2 [-+]?{dig}*\.{dig}+([eE][-+]?{dig}+)? +number {num1}|{num2} + +%% + +{ws} /* skip blanks and tabs */ + +"/*" { + int c; + + while((c = yyinput()) != 0) + { + if(c == '\n') + ++mylineno; + + else if(c == '*') + { + if((c = yyinput()) == '/') + break; + else + unput(c); + } + } + } + +{number} cout << "number " << YYText() << '\n'; + +\n mylineno++; + +{name} cout << "name " << YYText() << '\n'; + +{string} cout << "string " << YYText() << '\n'; + +%% + +int main( int /* argc */, char** /* argv */ ) + { + FlexLexer* lexer = new yyFlexLexer; + while(lexer->yylex() != 0) + ; + return 0; + } diff --git a/MISC/texinfo/flex.info b/MISC/texinfo/flex.info new file mode 100644 index 0000000..9269418 --- /dev/null +++ b/MISC/texinfo/flex.info @@ -0,0 +1,2951 @@ +This is Info file flex.info, produced by Makeinfo-1.55 from the input +file flex.texi. + +START-INFO-DIR-ENTRY +* Flex: (flex). A fast scanner generator. +END-INFO-DIR-ENTRY + + This file documents Flex. + + Copyright (c) 1990 The Regents of the University of California. All +rights reserved. + + This code is derived from software contributed to Berkeley by Vern +Paxson. + + The United States Government has rights in this work pursuant to +contract no. DE-AC03-76SF00098 between the United States Department of +Energy and the University of California. + + Redistribution and use in source and binary forms with or without +modification are permitted provided that: (1) source distributions +retain this entire copyright notice and comment, and (2) distributions +including binaries display the following acknowledgement: "This +product includes software developed by the University of California, +Berkeley and its contributors" in the documentation or other materials +provided with the distribution and in all advertising materials +mentioning features or use of this software. Neither the name of the +University nor the names of its contributors may be used to endorse or +promote products derived from this software without specific prior +written permission. + + THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + + +File: flex.info, Node: Top, Next: Name, Prev: (dir), Up: (dir) + +flex +**** + + This manual documents `flex'. It covers release 2.5. + +* Menu: + +* Name:: Name +* Synopsis:: Synopsis +* Overview:: Overview +* Description:: Description +* Examples:: Some simple examples +* Format:: Format of the input file +* Patterns:: Patterns +* Matching:: How the input is matched +* Actions:: Actions +* Generated scanner:: The generated scanner +* Start conditions:: Start conditions +* Multiple buffers:: Multiple input buffers +* End-of-file rules:: End-of-file rules +* Miscellaneous:: Miscellaneous macros +* User variables:: Values available to the user +* YACC interface:: Interfacing with `yacc' +* Options:: Options +* Performance:: Performance considerations +* C++:: Generating C++ scanners +* Incompatibilities:: Incompatibilities with `lex' and POSIX +* Diagnostics:: Diagnostics +* Files:: Files +* Deficiencies:: Deficiencies / Bugs +* See also:: See also +* Author:: Author + + +File: flex.info, Node: Name, Next: Synopsis, Prev: Top, Up: Top + +Name +==== + + flex - fast lexical analyzer generator + + +File: flex.info, Node: Synopsis, Next: Overview, Prev: Name, Up: Top + +Synopsis +======== + + flex [-bcdfhilnpstvwBFILTV78+? -C[aefFmr] -ooutput -Pprefix -Sskeleton] + [--help --version] [FILENAME ...] + + +File: flex.info, Node: Overview, Next: Description, Prev: Synopsis, Up: Top + +Overview +======== + + This manual describes `flex', a tool for generating programs that +perform pattern-matching on text. The manual includes both tutorial +and reference sections: + +Description + a brief overview of the tool + +Some Simple Examples +Format Of The Input File +Patterns + the extended regular expressions used by flex + +How The Input Is Matched + the rules for determining what has been matched + +Actions + how to specify what to do when a pattern is matched + +The Generated Scanner + details regarding the scanner that flex produces; how to control + the input source + +Start Conditions + introducing context into your scanners, and managing + "mini-scanners" + +Multiple Input Buffers + how to manipulate multiple input sources; how to scan from strings + instead of files + +End-of-file Rules + special rules for matching the end of the input + +Miscellaneous Macros + a summary of macros available to the actions + +Values Available To The User + a summary of values available to the actions + +Interfacing With Yacc + connecting flex scanners together with yacc parsers + +Options + flex command-line options, and the "%option" directive + +Performance Considerations + how to make your scanner go as fast as possible + +Generating C++ Scanners + the (experimental) facility for generating C++ scanner classes + +Incompatibilities With Lex And POSIX + how flex differs from AT&T lex and the POSIX lex standard + +Diagnostics + those error messages produced by flex (or scanners it generates) + whose meanings might not be apparent + +Files + files used by flex + +Deficiencies / Bugs + known problems with flex + +See Also + other documentation, related tools + +Author + includes contact information + + +File: flex.info, Node: Description, Next: Examples, Prev: Overview, Up: Top + +Description +=========== + + `flex' is a tool for generating "scanners": programs which +recognized lexical patterns in text. `flex' reads the given input +files, or its standard input if no file names are given, for a +description of a scanner to generate. The description is in the form +of pairs of regular expressions and C code, called "rules". `flex' +generates as output a C source file, `lex.yy.c', which defines a +routine `yylex()'. This file is compiled and linked with the `-lfl' +library to produce an executable. When the executable is run, it +analyzes its input for occurrences of the regular expressions. +Whenever it finds one, it executes the corresponding C code. + + +File: flex.info, Node: Examples, Next: Format, Prev: Description, Up: Top + +Some simple examples +==================== + + First some simple examples to get the flavor of how one uses `flex'. +The following `flex' input specifies a scanner which whenever it +encounters the string "username" will replace it with the user's login +name: + + %% + username printf( "%s", getlogin() ); + + By default, any text not matched by a `flex' scanner is copied to +the output, so the net effect of this scanner is to copy its input file +to its output with each occurrence of "username" expanded. In this +input, there is just one rule. "username" is the PATTERN and the +"printf" is the ACTION. The "%%" marks the beginning of the rules. + + Here's another simple example: + + int num_lines = 0, num_chars = 0; + + %% + \n ++num_lines; ++num_chars; + . ++num_chars; + + %% + main() + { + yylex(); + printf( "# of lines = %d, # of chars = %d\n", + num_lines, num_chars ); + } + + This scanner counts the number of characters and the number of lines +in its input (it produces no output other than the final report on the +counts). The first line declares two globals, "num_lines" and +"num_chars", which are accessible both inside `yylex()' and in the +`main()' routine declared after the second "%%". There are two rules, +one which matches a newline ("\n") and increments both the line count +and the character count, and one which matches any character other than +a newline (indicated by the "." regular expression). + + A somewhat more complicated example: + + /* scanner for a toy Pascal-like language */ + + %{ + /* need this for the call to atof() below */ + #include <math.h> + %} + + DIGIT [0-9] + ID [a-z][a-z0-9]* + + %% + + {DIGIT}+ { + printf( "An integer: %s (%d)\n", yytext, + atoi( yytext ) ); + } + + {DIGIT}+"."{DIGIT}* { + printf( "A float: %s (%g)\n", yytext, + atof( yytext ) ); + } + + if|then|begin|end|procedure|function { + printf( "A keyword: %s\n", yytext ); + } + + {ID} printf( "An identifier: %s\n", yytext ); + + "+"|"-"|"*"|"/" printf( "An operator: %s\n", yytext ); + + "{"[^}\n]*"}" /* eat up one-line comments */ + + [ \t\n]+ /* eat up whitespace */ + + . printf( "Unrecognized character: %s\n", yytext ); + + %% + + main( argc, argv ) + int argc; + char **argv; + { + ++argv, --argc; /* skip over program name */ + if ( argc > 0 ) + yyin = fopen( argv[0], "r" ); + else + yyin = stdin; + + yylex(); + } + + This is the beginnings of a simple scanner for a language like +Pascal. It identifies different types of TOKENS and reports on what it +has seen. + + The details of this example will be explained in the following +sections. + + +File: flex.info, Node: Format, Next: Patterns, Prev: Examples, Up: Top + +Format of the input file +======================== + + The `flex' input file consists of three sections, separated by a +line with just `%%' in it: + + definitions + %% + rules + %% + user code + + The "definitions" section contains declarations of simple "name" +definitions to simplify the scanner specification, and declarations of +"start conditions", which are explained in a later section. Name +definitions have the form: + + name definition + + The "name" is a word beginning with a letter or an underscore ('_') +followed by zero or more letters, digits, '_', or '-' (dash). The +definition is taken to begin at the first non-white-space character +following the name and continuing to the end of the line. The +definition can subsequently be referred to using "{name}", which will +expand to "(definition)". For example, + + DIGIT [0-9] + ID [a-z][a-z0-9]* + +defines "DIGIT" to be a regular expression which matches a single +digit, and "ID" to be a regular expression which matches a letter +followed by zero-or-more letters-or-digits. A subsequent reference to + + {DIGIT}+"."{DIGIT}* + +is identical to + + ([0-9])+"."([0-9])* + +and matches one-or-more digits followed by a '.' followed by +zero-or-more digits. + + The RULES section of the `flex' input contains a series of rules of +the form: + + pattern action + +where the pattern must be unindented and the action must begin on the +same line. + + See below for a further description of patterns and actions. + + Finally, the user code section is simply copied to `lex.yy.c' +verbatim. It is used for companion routines which call or are called +by the scanner. The presence of this section is optional; if it is +missing, the second `%%' in the input file may be skipped, too. + + In the definitions and rules sections, any *indented* text or text +enclosed in `%{' and `%}' is copied verbatim to the output (with the +`%{}''s removed). The `%{}''s must appear unindented on lines by +themselves. + + In the rules section, any indented or %{} text appearing before the +first rule may be used to declare variables which are local to the +scanning routine and (after the declarations) code which is to be +executed whenever the scanning routine is entered. Other indented or +%{} text in the rule section is still copied to the output, but its +meaning is not well-defined and it may well cause compile-time errors +(this feature is present for `POSIX' compliance; see below for other +such features). + + In the definitions section (but not in the rules section), an +unindented comment (i.e., a line beginning with "/*") is also copied +verbatim to the output up to the next "*/". + + +File: flex.info, Node: Patterns, Next: Matching, Prev: Format, Up: Top + +Patterns +======== + + The patterns in the input are written using an extended set of +regular expressions. These are: + +`x' + match the character `x' + +`.' + any character (byte) except newline + +`[xyz]' + a "character class"; in this case, the pattern matches either an + `x', a `y', or a `z' + +`[abj-oZ]' + a "character class" with a range in it; matches an `a', a `b', any + letter from `j' through `o', or a `Z' + +`[^A-Z]' + a "negated character class", i.e., any character but those in the + class. In this case, any character EXCEPT an uppercase letter. + +`[^A-Z\n]' + any character EXCEPT an uppercase letter or a newline + +`R*' + zero or more R's, where R is any regular expression + +`R+' + one or more R's + +`R?' + zero or one R's (that is, "an optional R") + +`R{2,5}' + anywhere from two to five R's + +`R{2,}' + two or more R's + +`R{4}' + exactly 4 R's + +`{NAME}' + the expansion of the "NAME" definition (see above) + +`"[xyz]\"foo"' + the literal string: `[xyz]"foo' + +`\X' + if X is an `a', `b', `f', `n', `r', `t', or `v', then the ANSI-C + interpretation of \X. Otherwise, a literal `X' (used to escape + operators such as `*') + +`\0' + a NUL character (ASCII code 0) + +`\123' + the character with octal value 123 + +`\x2a' + the character with hexadecimal value `2a' + +`(R)' + match an R; parentheses are used to override precedence (see below) + +`RS' + the regular expression R followed by the regular expression S; + called "concatenation" + +`R|S' + either an R or an S + +`R/S' + an R but only if it is followed by an S. The text matched by S is + included when determining whether this rule is the "longest + match", but is then returned to the input before the action is + executed. So the action only sees the text matched by R. This + type of pattern is called "trailing context". (There are some + combinations of `R/S' that `flex' cannot match correctly; see + notes in the Deficiencies / Bugs section below regarding + "dangerous trailing context".) + +`^R' + an R, but only at the beginning of a line (i.e., which just + starting to scan, or right after a newline has been scanned). + +`R$' + an R, but only at the end of a line (i.e., just before a newline). + Equivalent to "R/\n". + + Note that flex's notion of "newline" is exactly whatever the C + compiler used to compile flex interprets '\n' as; in particular, + on some DOS systems you must either filter out \r's in the input + yourself, or explicitly use R/\r\n for "r$". + +`<S>R' + an R, but only in start condition S (see below for discussion of + start conditions) <S1,S2,S3>R same, but in any of start conditions + S1, S2, or S3 + +`<*>R' + an R in any start condition, even an exclusive one. + +`<<EOF>>' + an end-of-file <S1,S2><<EOF>> an end-of-file when in start + condition S1 or S2 + + Note that inside of a character class, all regular expression +operators lose their special meaning except escape ('\') and the +character class operators, '-', ']', and, at the beginning of the +class, '^'. + + The regular expressions listed above are grouped according to +precedence, from highest precedence at the top to lowest at the bottom. +Those grouped together have equal precedence. For example, + + foo|bar* + +is the same as + + (foo)|(ba(r*)) + +since the '*' operator has higher precedence than concatenation, and +concatenation higher than alternation ('|'). This pattern therefore +matches *either* the string "foo" *or* the string "ba" followed by +zero-or-more r's. To match "foo" or zero-or-more "bar"'s, use: + + foo|(bar)* + +and to match zero-or-more "foo"'s-or-"bar"'s: + + (foo|bar)* + + In addition to characters and ranges of characters, character +classes can also contain character class "expressions". These are +expressions enclosed inside `[': and `:'] delimiters (which themselves +must appear between the '[' and ']' of the character class; other +elements may occur inside the character class, too). The valid +expressions are: + + [:alnum:] [:alpha:] [:blank:] + [:cntrl:] [:digit:] [:graph:] + [:lower:] [:print:] [:punct:] + [:space:] [:upper:] [:xdigit:] + + These expressions all designate a set of characters equivalent to +the corresponding standard C `isXXX' function. For example, +`[:alnum:]' designates those characters for which `isalnum()' returns +true - i.e., any alphabetic or numeric. Some systems don't provide +`isblank()', so flex defines `[:blank:]' as a blank or a tab. + + For example, the following character classes are all equivalent: + + [[:alnum:]] + [[:alpha:][:digit:] + [[:alpha:]0-9] + [a-zA-Z0-9] + + If your scanner is case-insensitive (the `-i' flag), then +`[:upper:]' and `[:lower:]' are equivalent to `[:alpha:]'. + + Some notes on patterns: + + - A negated character class such as the example "[^A-Z]" above *will + match a newline* unless "\n" (or an equivalent escape sequence) is + one of the characters explicitly present in the negated character + class (e.g., "[^A-Z\n]"). This is unlike how many other regular + expression tools treat negated character classes, but + unfortunately the inconsistency is historically entrenched. + Matching newlines means that a pattern like [^"]* can match the + entire input unless there's another quote in the input. + + - A rule can have at most one instance of trailing context (the '/' + operator or the '$' operator). The start condition, '^', and + "<<EOF>>" patterns can only occur at the beginning of a pattern, + and, as well as with '/' and '$', cannot be grouped inside + parentheses. A '^' which does not occur at the beginning of a + rule or a '$' which does not occur at the end of a rule loses its + special properties and is treated as a normal character. + + The following are illegal: + + foo/bar$ + <sc1>foo<sc2>bar + + Note that the first of these, can be written "foo/bar\n". + + The following will result in '$' or '^' being treated as a normal + character: + + foo|(bar$) + foo|^bar + + If what's wanted is a "foo" or a bar-followed-by-a-newline, the + following could be used (the special '|' action is explained + below): + + foo | + bar$ /* action goes here */ + + A similar trick will work for matching a foo or a + bar-at-the-beginning-of-a-line. + + +File: flex.info, Node: Matching, Next: Actions, Prev: Patterns, Up: Top + +How the input is matched +======================== + + When the generated scanner is run, it analyzes its input looking for +strings which match any of its patterns. If it finds more than one +match, it takes the one matching the most text (for trailing context +rules, this includes the length of the trailing part, even though it +will then be returned to the input). If it finds two or more matches +of the same length, the rule listed first in the `flex' input file is +chosen. + + Once the match is determined, the text corresponding to the match +(called the TOKEN) is made available in the global character pointer +`yytext', and its length in the global integer `yyleng'. The ACTION +corresponding to the matched pattern is then executed (a more detailed +description of actions follows), and then the remaining input is +scanned for another match. + + If no match is found, then the "default rule" is executed: the next +character in the input is considered matched and copied to the standard +output. Thus, the simplest legal `flex' input is: + + %% + + which generates a scanner that simply copies its input (one +character at a time) to its output. + + Note that `yytext' can be defined in two different ways: either as a +character *pointer* or as a character *array*. You can control which +definition `flex' uses by including one of the special directives +`%pointer' or `%array' in the first (definitions) section of your flex +input. The default is `%pointer', unless you use the `-l' lex +compatibility option, in which case `yytext' will be an array. The +advantage of using `%pointer' is substantially faster scanning and no +buffer overflow when matching very large tokens (unless you run out of +dynamic memory). The disadvantage is that you are restricted in how +your actions can modify `yytext' (see the next section), and calls to +the `unput()' function destroys the present contents of `yytext', which +can be a considerable porting headache when moving between different +`lex' versions. + + The advantage of `%array' is that you can then modify `yytext' to +your heart's content, and calls to `unput()' do not destroy `yytext' +(see below). Furthermore, existing `lex' programs sometimes access +`yytext' externally using declarations of the form: + extern char yytext[]; + This definition is erroneous when used with `%pointer', but correct +for `%array'. + + `%array' defines `yytext' to be an array of `YYLMAX' characters, +which defaults to a fairly large value. You can change the size by +simply #define'ing `YYLMAX' to a different value in the first section +of your `flex' input. As mentioned above, with `%pointer' yytext grows +dynamically to accommodate large tokens. While this means your +`%pointer' scanner can accommodate very large tokens (such as matching +entire blocks of comments), bear in mind that each time the scanner +must resize `yytext' it also must rescan the entire token from the +beginning, so matching such tokens can prove slow. `yytext' presently +does *not* dynamically grow if a call to `unput()' results in too much +text being pushed back; instead, a run-time error results. + + Also note that you cannot use `%array' with C++ scanner classes (the +`c++' option; see below). + + +File: flex.info, Node: Actions, Next: Generated scanner, Prev: Matching, Up: Top + +Actions +======= + + Each pattern in a rule has a corresponding action, which can be any +arbitrary C statement. The pattern ends at the first non-escaped +whitespace character; the remainder of the line is its action. If the +action is empty, then when the pattern is matched the input token is +simply discarded. For example, here is the specification for a program +which deletes all occurrences of "zap me" from its input: + + %% + "zap me" + + (It will copy all other characters in the input to the output since +they will be matched by the default rule.) + + Here is a program which compresses multiple blanks and tabs down to +a single blank, and throws away whitespace found at the end of a line: + + %% + [ \t]+ putchar( ' ' ); + [ \t]+$ /* ignore this token */ + + If the action contains a '{', then the action spans till the +balancing '}' is found, and the action may cross multiple lines. +`flex' knows about C strings and comments and won't be fooled by braces +found within them, but also allows actions to begin with `%{' and will +consider the action to be all the text up to the next `%}' (regardless +of ordinary braces inside the action). + + An action consisting solely of a vertical bar ('|') means "same as +the action for the next rule." See below for an illustration. + + Actions can include arbitrary C code, including `return' statements +to return a value to whatever routine called `yylex()'. Each time +`yylex()' is called it continues processing tokens from where it last +left off until it either reaches the end of the file or executes a +return. + + Actions are free to modify `yytext' except for lengthening it +(adding characters to its end-these will overwrite later characters in +the input stream). This however does not apply when using `%array' +(see above); in that case, `yytext' may be freely modified in any way. + + Actions are free to modify `yyleng' except they should not do so if +the action also includes use of `yymore()' (see below). + + There are a number of special directives which can be included +within an action: + + - `ECHO' copies yytext to the scanner's output. + + - `BEGIN' followed by the name of a start condition places the + scanner in the corresponding start condition (see below). + + - `REJECT' directs the scanner to proceed on to the "second best" + rule which matched the input (or a prefix of the input). The rule + is chosen as described above in "How the Input is Matched", and + `yytext' and `yyleng' set up appropriately. It may either be one + which matched as much text as the originally chosen rule but came + later in the `flex' input file, or one which matched less text. + For example, the following will both count the words in the input + and call the routine special() whenever "frob" is seen: + + int word_count = 0; + %% + + frob special(); REJECT; + [^ \t\n]+ ++word_count; + + Without the `REJECT', any "frob"'s in the input would not be + counted as words, since the scanner normally executes only one + action per token. Multiple `REJECT's' are allowed, each one + finding the next best choice to the currently active rule. For + example, when the following scanner scans the token "abcd", it + will write "abcdabcaba" to the output: + + %% + a | + ab | + abc | + abcd ECHO; REJECT; + .|\n /* eat up any unmatched character */ + + (The first three rules share the fourth's action since they use + the special '|' action.) `REJECT' is a particularly expensive + feature in terms of scanner performance; if it is used in *any* of + the scanner's actions it will slow down *all* of the scanner's + matching. Furthermore, `REJECT' cannot be used with the `-Cf' or + `-CF' options (see below). + + Note also that unlike the other special actions, `REJECT' is a + *branch*; code immediately following it in the action will *not* + be executed. + + - `yymore()' tells the scanner that the next time it matches a rule, + the corresponding token should be *appended* onto the current + value of `yytext' rather than replacing it. For example, given + the input "mega-kludge" the following will write + "mega-mega-kludge" to the output: + + %% + mega- ECHO; yymore(); + kludge ECHO; + + First "mega-" is matched and echoed to the output. Then "kludge" + is matched, but the previous "mega-" is still hanging around at + the beginning of `yytext' so the `ECHO' for the "kludge" rule will + actually write "mega-kludge". + + Two notes regarding use of `yymore()'. First, `yymore()' depends on +the value of `yyleng' correctly reflecting the size of the current +token, so you must not modify `yyleng' if you are using `yymore()'. +Second, the presence of `yymore()' in the scanner's action entails a +minor performance penalty in the scanner's matching speed. + + - `yyless(n)' returns all but the first N characters of the current + token back to the input stream, where they will be rescanned when + the scanner looks for the next match. `yytext' and `yyleng' are + adjusted appropriately (e.g., `yyleng' will now be equal to N ). + For example, on the input "foobar" the following will write out + "foobarbar": + + %% + foobar ECHO; yyless(3); + [a-z]+ ECHO; + + An argument of 0 to `yyless' will cause the entire current input + string to be scanned again. Unless you've changed how the scanner + will subsequently process its input (using `BEGIN', for example), + this will result in an endless loop. + + Note that `yyless' is a macro and can only be used in the flex + input file, not from other source files. + + - `unput(c)' puts the character `c' back onto the input stream. It + will be the next character scanned. The following action will + take the current token and cause it to be rescanned enclosed in + parentheses. + + { + int i; + /* Copy yytext because unput() trashes yytext */ + char *yycopy = strdup( yytext ); + unput( ')' ); + for ( i = yyleng - 1; i >= 0; --i ) + unput( yycopy[i] ); + unput( '(' ); + free( yycopy ); + } + + Note that since each `unput()' puts the given character back at + the *beginning* of the input stream, pushing back strings must be + done back-to-front. An important potential problem when using + `unput()' is that if you are using `%pointer' (the default), a + call to `unput()' *destroys* the contents of `yytext', starting + with its rightmost character and devouring one character to the + left with each call. If you need the value of yytext preserved + after a call to `unput()' (as in the above example), you must + either first copy it elsewhere, or build your scanner using + `%array' instead (see How The Input Is Matched). + + Finally, note that you cannot put back `EOF' to attempt to mark + the input stream with an end-of-file. + + - `input()' reads the next character from the input stream. For + example, the following is one way to eat up C comments: + + %% + "/*" { + register int c; + + for ( ; ; ) + { + while ( (c = input()) != '*' && + c != EOF ) + ; /* eat up text of comment */ + + if ( c == '*' ) + { + while ( (c = input()) == '*' ) + ; + if ( c == '/' ) + break; /* found the end */ + } + + if ( c == EOF ) + { + error( "EOF in comment" ); + break; + } + } + } + + (Note that if the scanner is compiled using `C++', then `input()' + is instead referred to as `yyinput()', in order to avoid a name + clash with the `C++' stream by the name of `input'.) + + - YY_FLUSH_BUFFER flushes the scanner's internal buffer so that the + next time the scanner attempts to match a token, it will first + refill the buffer using `YY_INPUT' (see The Generated Scanner, + below). This action is a special case of the more general + `yy_flush_buffer()' function, described below in the section + Multiple Input Buffers. + + - `yyterminate()' can be used in lieu of a return statement in an + action. It terminates the scanner and returns a 0 to the + scanner's caller, indicating "all done". By default, + `yyterminate()' is also called when an end-of-file is encountered. + It is a macro and may be redefined. + + +File: flex.info, Node: Generated scanner, Next: Start conditions, Prev: Actions, Up: Top + +The generated scanner +===================== + + The output of `flex' is the file `lex.yy.c', which contains the +scanning routine `yylex()', a number of tables used by it for matching +tokens, and a number of auxiliary routines and macros. By default, +`yylex()' is declared as follows: + + int yylex() + { + ... various definitions and the actions in here ... + } + + (If your environment supports function prototypes, then it will be +"int yylex( void )".) This definition may be changed by defining +the "YY_DECL" macro. For example, you could use: + + #define YY_DECL float lexscan( a, b ) float a, b; + + to give the scanning routine the name `lexscan', returning a float, +and taking two floats as arguments. Note that if you give arguments to +the scanning routine using a K&R-style/non-prototyped function +declaration, you must terminate the definition with a semi-colon (`;'). + + Whenever `yylex()' is called, it scans tokens from the global input +file `yyin' (which defaults to stdin). It continues until it either +reaches an end-of-file (at which point it returns the value 0) or one +of its actions executes a `return' statement. + + If the scanner reaches an end-of-file, subsequent calls are undefined +unless either `yyin' is pointed at a new input file (in which case +scanning continues from that file), or `yyrestart()' is called. +`yyrestart()' takes one argument, a `FILE *' pointer (which can be nil, +if you've set up `YY_INPUT' to scan from a source other than `yyin'), +and initializes `yyin' for scanning from that file. Essentially there +is no difference between just assigning `yyin' to a new input file or +using `yyrestart()' to do so; the latter is available for compatibility +with previous versions of `flex', and because it can be used to switch +input files in the middle of scanning. It can also be used to throw +away the current input buffer, by calling it with an argument of +`yyin'; but better is to use `YY_FLUSH_BUFFER' (see above). Note that +`yyrestart()' does *not* reset the start condition to `INITIAL' (see +Start Conditions, below). + + If `yylex()' stops scanning due to executing a `return' statement in +one of the actions, the scanner may then be called again and it will +resume scanning where it left off. + + By default (and for purposes of efficiency), the scanner uses +block-reads rather than simple `getc()' calls to read characters from +`yyin'. The nature of how it gets its input can be controlled by +defining the `YY_INPUT' macro. YY_INPUT's calling sequence is +"YY_INPUT(buf,result,max_size)". Its action is to place up to MAX_SIZE +characters in the character array BUF and return in the integer +variable RESULT either the number of characters read or the constant +YY_NULL (0 on Unix systems) to indicate EOF. The default YY_INPUT +reads from the global file-pointer "yyin". + + A sample definition of YY_INPUT (in the definitions section of the +input file): + + %{ + #define YY_INPUT(buf,result,max_size) \ + { \ + int c = getchar(); \ + result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \ + } + %} + + This definition will change the input processing to occur one +character at a time. + + When the scanner receives an end-of-file indication from YY_INPUT, +it then checks the `yywrap()' function. If `yywrap()' returns false +(zero), then it is assumed that the function has gone ahead and set up +`yyin' to point to another input file, and scanning continues. If it +returns true (non-zero), then the scanner terminates, returning 0 to +its caller. Note that in either case, the start condition remains +unchanged; it does *not* revert to `INITIAL'. + + If you do not supply your own version of `yywrap()', then you must +either use `%option noyywrap' (in which case the scanner behaves as +though `yywrap()' returned 1), or you must link with `-lfl' to obtain +the default version of the routine, which always returns 1. + + Three routines are available for scanning from in-memory buffers +rather than files: `yy_scan_string()', `yy_scan_bytes()', and +`yy_scan_buffer()'. See the discussion of them below in the section +Multiple Input Buffers. + + The scanner writes its `ECHO' output to the `yyout' global (default, +stdout), which may be redefined by the user simply by assigning it to +some other `FILE' pointer. + + +File: flex.info, Node: Start conditions, Next: Multiple buffers, Prev: Generated scanner, Up: Top + +Start conditions +================ + + `flex' provides a mechanism for conditionally activating rules. Any +rule whose pattern is prefixed with "<sc>" will only be active when the +scanner is in the start condition named "sc". For example, + + <STRING>[^"]* { /* eat up the string body ... */ + ... + } + +will be active only when the scanner is in the "STRING" start +condition, and + + <INITIAL,STRING,QUOTE>\. { /* handle an escape ... */ + ... + } + +will be active only when the current start condition is either +"INITIAL", "STRING", or "QUOTE". + + Start conditions are declared in the definitions (first) section of +the input using unindented lines beginning with either `%s' or `%x' +followed by a list of names. The former declares *inclusive* start +conditions, the latter *exclusive* start conditions. A start condition +is activated using the `BEGIN' action. Until the next `BEGIN' action is +executed, rules with the given start condition will be active and rules +with other start conditions will be inactive. If the start condition +is *inclusive*, then rules with no start conditions at all will also be +active. If it is *exclusive*, then *only* rules qualified with the +start condition will be active. A set of rules contingent on the same +exclusive start condition describe a scanner which is independent of +any of the other rules in the `flex' input. Because of this, exclusive +start conditions make it easy to specify "mini-scanners" which scan +portions of the input that are syntactically different from the rest +(e.g., comments). + + If the distinction between inclusive and exclusive start conditions +is still a little vague, here's a simple example illustrating the +connection between the two. The set of rules: + + %s example + %% + + <example>foo do_something(); + + bar something_else(); + +is equivalent to + + %x example + %% + + <example>foo do_something(); + + <INITIAL,example>bar something_else(); + + Without the `<INITIAL,example>' qualifier, the `bar' pattern in the +second example wouldn't be active (i.e., couldn't match) when in start +condition `example'. If we just used `<example>' to qualify `bar', +though, then it would only be active in `example' and not in `INITIAL', +while in the first example it's active in both, because in the first +example the `example' starting condition is an *inclusive* (`%s') start +condition. + + Also note that the special start-condition specifier `<*>' matches +every start condition. Thus, the above example could also have been +written; + + %x example + %% + + <example>foo do_something(); + + <*>bar something_else(); + + The default rule (to `ECHO' any unmatched character) remains active +in start conditions. It is equivalent to: + + <*>.|\\n ECHO; + + `BEGIN(0)' returns to the original state where only the rules with +no start conditions are active. This state can also be referred to as +the start-condition "INITIAL", so `BEGIN(INITIAL)' is equivalent to +`BEGIN(0)'. (The parentheses around the start condition name are not +required but are considered good style.) + + `BEGIN' actions can also be given as indented code at the beginning +of the rules section. For example, the following will cause the +scanner to enter the "SPECIAL" start condition whenever `yylex()' is +called and the global variable `enter_special' is true: + + int enter_special; + + %x SPECIAL + %% + if ( enter_special ) + BEGIN(SPECIAL); + + <SPECIAL>blahblahblah + ...more rules follow... + + To illustrate the uses of start conditions, here is a scanner which +provides two different interpretations of a string like "123.456". By +default it will treat it as as three tokens, the integer "123", a dot +('.'), and the integer "456". But if the string is preceded earlier in +the line by the string "expect-floats" it will treat it as a single +token, the floating-point number 123.456: + + %{ + #include <math.h> + %} + %s expect + + %% + expect-floats BEGIN(expect); + + <expect>[0-9]+"."[0-9]+ { + printf( "found a float, = %f\n", + atof( yytext ) ); + } + <expect>\n { + /* that's the end of the line, so + * we need another "expect-number" + * before we'll recognize any more + * numbers + */ + BEGIN(INITIAL); + } + + [0-9]+ { + + Version 2.5 December 1994 18 + + printf( "found an integer, = %d\n", + atoi( yytext ) ); + } + + "." printf( "found a dot\n" ); + + Here is a scanner which recognizes (and discards) C comments while +maintaining a count of the current input line. + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + This scanner goes to a bit of trouble to match as much text as +possible with each rule. In general, when attempting to write a +high-speed scanner try to match as much possible in each rule, as it's +a big win. + + Note that start-conditions names are really integer values and can +be stored as such. Thus, the above could be extended in the following +fashion: + + %x comment foo + %% + int line_num = 1; + int comment_caller; + + "/*" { + comment_caller = INITIAL; + BEGIN(comment); + } + + ... + + <foo>"/*" { + comment_caller = foo; + BEGIN(comment); + } + + <comment>[^*\n]* /* eat anything that's not a '*' */ + <comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(comment_caller); + + Furthermore, you can access the current start condition using the +integer-valued `YY_START' macro. For example, the above assignments to +`comment_caller' could instead be written + + comment_caller = YY_START; + + Flex provides `YYSTATE' as an alias for `YY_START' (since that is +what's used by AT&T `lex'). + + Note that start conditions do not have their own name-space; %s's +and %x's declare names in the same fashion as #define's. + + Finally, here's an example of how to match C-style quoted strings +using exclusive start conditions, including expanded escape sequences +(but not including checking for a string that's too long): + + %x str + + %% + char string_buf[MAX_STR_CONST]; + char *string_buf_ptr; + + \" string_buf_ptr = string_buf; BEGIN(str); + + <str>\" { /* saw closing quote - all done */ + BEGIN(INITIAL); + *string_buf_ptr = '\0'; + /* return string constant token type and + * value to parser + */ + } + + <str>\n { + /* error - unterminated string constant */ + /* generate error message */ + } + + <str>\\[0-7]{1,3} { + /* octal escape sequence */ + int result; + + (void) sscanf( yytext + 1, "%o", &result ); + + if ( result > 0xff ) + /* error, constant is out-of-bounds */ + + *string_buf_ptr++ = result; + } + + <str>\\[0-9]+ { + /* generate error - bad escape sequence; something + * like '\48' or '\0777777' + */ + } + + <str>\\n *string_buf_ptr++ = '\n'; + <str>\\t *string_buf_ptr++ = '\t'; + <str>\\r *string_buf_ptr++ = '\r'; + <str>\\b *string_buf_ptr++ = '\b'; + <str>\\f *string_buf_ptr++ = '\f'; + + <str>\\(.|\n) *string_buf_ptr++ = yytext[1]; + + <str>[^\\\n\"]+ { + char *yptr = yytext; + + while ( *yptr ) + *string_buf_ptr++ = *yptr++; + } + + Often, such as in some of the examples above, you wind up writing a +whole bunch of rules all preceded by the same start condition(s). Flex +makes this a little easier and cleaner by introducing a notion of start +condition "scope". A start condition scope is begun with: + + <SCs>{ + +where SCs is a list of one or more start conditions. Inside the start +condition scope, every rule automatically has the prefix `<SCs>' +applied to it, until a `}' which matches the initial `{'. So, for +example, + + <ESC>{ + "\\n" return '\n'; + "\\r" return '\r'; + "\\f" return '\f'; + "\\0" return '\0'; + } + +is equivalent to: + + <ESC>"\\n" return '\n'; + <ESC>"\\r" return '\r'; + <ESC>"\\f" return '\f'; + <ESC>"\\0" return '\0'; + + Start condition scopes may be nested. + + Three routines are available for manipulating stacks of start +conditions: + +`void yy_push_state(int new_state)' + pushes the current start condition onto the top of the start + condition stack and switches to NEW_STATE as though you had used + `BEGIN new_state' (recall that start condition names are also + integers). + +`void yy_pop_state()' + pops the top of the stack and switches to it via `BEGIN'. + +`int yy_top_state()' + returns the top of the stack without altering the stack's contents. + + The start condition stack grows dynamically and so has no built-in +size limitation. If memory is exhausted, program execution aborts. + + To use start condition stacks, your scanner must include a `%option +stack' directive (see Options below). + + +File: flex.info, Node: Multiple buffers, Next: End-of-file rules, Prev: Start conditions, Up: Top + +Multiple input buffers +====================== + + Some scanners (such as those which support "include" files) require +reading from several input streams. As `flex' scanners do a large +amount of buffering, one cannot control where the next input will be +read from by simply writing a `YY_INPUT' which is sensitive to the +scanning context. `YY_INPUT' is only called when the scanner reaches +the end of its buffer, which may be a long time after scanning a +statement such as an "include" which requires switching the input +source. + + To negotiate these sorts of problems, `flex' provides a mechanism +for creating and switching between multiple input buffers. An input +buffer is created by using: + + YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) + +which takes a `FILE' pointer and a size and creates a buffer associated +with the given file and large enough to hold SIZE characters (when in +doubt, use `YY_BUF_SIZE' for the size). It returns a `YY_BUFFER_STATE' +handle, which may then be passed to other routines (see below). The +`YY_BUFFER_STATE' type is a pointer to an opaque `struct' +`yy_buffer_state' structure, so you may safely initialize +YY_BUFFER_STATE variables to `((YY_BUFFER_STATE) 0)' if you wish, and +also refer to the opaque structure in order to correctly declare input +buffers in source files other than that of your scanner. Note that the +`FILE' pointer in the call to `yy_create_buffer' is only used as the +value of `yyin' seen by `YY_INPUT'; if you redefine `YY_INPUT' so it no +longer uses `yyin', then you can safely pass a nil `FILE' pointer to +`yy_create_buffer'. You select a particular buffer to scan from using: + + void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) + + switches the scanner's input buffer so subsequent tokens will come +from NEW_BUFFER. Note that `yy_switch_to_buffer()' may be used by +`yywrap()' to set things up for continued scanning, instead of opening +a new file and pointing `yyin' at it. Note also that switching input +sources via either `yy_switch_to_buffer()' or `yywrap()' does *not* +change the start condition. + + void yy_delete_buffer( YY_BUFFER_STATE buffer ) + +is used to reclaim the storage associated with a buffer. You can also +clear the current contents of a buffer using: + + void yy_flush_buffer( YY_BUFFER_STATE buffer ) + + This function discards the buffer's contents, so the next time the +scanner attempts to match a token from the buffer, it will first fill +the buffer anew using `YY_INPUT'. + + `yy_new_buffer()' is an alias for `yy_create_buffer()', provided for +compatibility with the C++ use of `new' and `delete' for creating and +destroying dynamic objects. + + Finally, the `YY_CURRENT_BUFFER' macro returns a `YY_BUFFER_STATE' +handle to the current buffer. + + Here is an example of using these features for writing a scanner +which expands include files (the `<<EOF>>' feature is discussed below): + + /* the "incl" state is used for picking up the name + * of an include file + */ + %x incl + + %{ + #define MAX_INCLUDE_DEPTH 10 + YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH]; + int include_stack_ptr = 0; + %} + + %% + include BEGIN(incl); + + [a-z]+ ECHO; + [^a-z\n]*\n? ECHO; + + <incl>[ \t]* /* eat the whitespace */ + <incl>[^ \t\n]+ { /* got the include file name */ + if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) + { + fprintf( stderr, "Includes nested too deeply" ); + exit( 1 ); + } + + include_stack[include_stack_ptr++] = + YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + + if ( ! yyin ) + error( ... ); + + yy_switch_to_buffer( + yy_create_buffer( yyin, YY_BUF_SIZE ) ); + + BEGIN(INITIAL); + } + + <<EOF>> { + if ( --include_stack_ptr < 0 ) + { + yyterminate(); + } + + else + { + yy_delete_buffer( YY_CURRENT_BUFFER ); + yy_switch_to_buffer( + include_stack[include_stack_ptr] ); + } + } + + Three routines are available for setting up input buffers for +scanning in-memory strings instead of files. All of them create a new +input buffer for scanning the string, and return a corresponding +`YY_BUFFER_STATE' handle (which you should delete with +`yy_delete_buffer()' when done with it). They also switch to the new +buffer using `yy_switch_to_buffer()', so the next call to `yylex()' will +start scanning the string. + +`yy_scan_string(const char *str)' + scans a NUL-terminated string. + +`yy_scan_bytes(const char *bytes, int len)' + scans `len' bytes (including possibly NUL's) starting at location + BYTES. + + Note that both of these functions create and scan a *copy* of the +string or bytes. (This may be desirable, since `yylex()' modifies the +contents of the buffer it is scanning.) You can avoid the copy by using: + +`yy_scan_buffer(char *base, yy_size_t size)' + which scans in place the buffer starting at BASE, consisting of + SIZE bytes, the last two bytes of which *must* be + `YY_END_OF_BUFFER_CHAR' (ASCII NUL). These last two bytes are not + scanned; thus, scanning consists of `base[0]' through + `base[size-2]', inclusive. + + If you fail to set up BASE in this manner (i.e., forget the final + two `YY_END_OF_BUFFER_CHAR' bytes), then `yy_scan_buffer()' + returns a nil pointer instead of creating a new input buffer. + + The type `yy_size_t' is an integral type to which you can cast an + integer expression reflecting the size of the buffer. + + +File: flex.info, Node: End-of-file rules, Next: Miscellaneous, Prev: Multiple buffers, Up: Top + +End-of-file rules +================= + + The special rule "<<EOF>>" indicates actions which are to be taken +when an end-of-file is encountered and yywrap() returns non-zero (i.e., +indicates no further files to process). The action must finish by +doing one of four things: + + - assigning `yyin' to a new input file (in previous versions of + flex, after doing the assignment you had to call the special + action `YY_NEW_FILE'; this is no longer necessary); + + - executing a `return' statement; + + - executing the special `yyterminate()' action; + + - or, switching to a new buffer using `yy_switch_to_buffer()' as + shown in the example above. + + <<EOF>> rules may not be used with other patterns; they may only be +qualified with a list of start conditions. If an unqualified <<EOF>> +rule is given, it applies to *all* start conditions which do not +already have <<EOF>> actions. To specify an <<EOF>> rule for only the +initial start condition, use + + <INITIAL><<EOF>> + + These rules are useful for catching things like unclosed comments. +An example: + + %x quote + %% + + ...other rules for dealing with quotes... + + <quote><<EOF>> { + error( "unterminated quote" ); + yyterminate(); + } + <<EOF>> { + if ( *++filelist ) + yyin = fopen( *filelist, "r" ); + else + yyterminate(); + } + + +File: flex.info, Node: Miscellaneous, Next: User variables, Prev: End-of-file rules, Up: Top + +Miscellaneous macros +==================== + + The macro `YY_USER_ACTION' can be defined to provide an action which +is always executed prior to the matched rule's action. For example, it +could be #define'd to call a routine to convert yytext to lower-case. +When `YY_USER_ACTION' is invoked, the variable `yy_act' gives the +number of the matched rule (rules are numbered starting with 1). +Suppose you want to profile how often each of your rules is matched. +The following would do the trick: + + #define YY_USER_ACTION ++ctr[yy_act] + + where `ctr' is an array to hold the counts for the different rules. +Note that the macro `YY_NUM_RULES' gives the total number of rules +(including the default rule, even if you use `-s', so a correct +declaration for `ctr' is: + + int ctr[YY_NUM_RULES]; + + The macro `YY_USER_INIT' may be defined to provide an action which +is always executed before the first scan (and before the scanner's +internal initializations are done). For example, it could be used to +call a routine to read in a data table or open a logging file. + + The macro `yy_set_interactive(is_interactive)' can be used to +control whether the current buffer is considered *interactive*. An +interactive buffer is processed more slowly, but must be used when the +scanner's input source is indeed interactive to avoid problems due to +waiting to fill buffers (see the discussion of the `-I' flag below). A +non-zero value in the macro invocation marks the buffer as interactive, +a zero value as non-interactive. Note that use of this macro overrides +`%option always-interactive' or `%option never-interactive' (see +Options below). `yy_set_interactive()' must be invoked prior to +beginning to scan the buffer that is (or is not) to be considered +interactive. + + The macro `yy_set_bol(at_bol)' can be used to control whether the +current buffer's scanning context for the next token match is done as +though at the beginning of a line. A non-zero macro argument makes +rules anchored with + + The macro `YY_AT_BOL()' returns true if the next token scanned from +the current buffer will have '^' rules active, false otherwise. + + In the generated scanner, the actions are all gathered in one large +switch statement and separated using `YY_BREAK', which may be +redefined. By default, it is simply a "break", to separate each rule's +action from the following rule's. Redefining `YY_BREAK' allows, for +example, C++ users to #define YY_BREAK to do nothing (while being very +careful that every rule ends with a "break" or a "return"!) to avoid +suffering from unreachable statement warnings where because a rule's +action ends with "return", the `YY_BREAK' is inaccessible. + + +File: flex.info, Node: User variables, Next: YACC interface, Prev: Miscellaneous, Up: Top + +Values available to the user +============================ + + This section summarizes the various values available to the user in +the rule actions. + + - `char *yytext' holds the text of the current token. It may be + modified but not lengthened (you cannot append characters to the + end). + + If the special directive `%array' appears in the first section of + the scanner description, then `yytext' is instead declared `char + yytext[YYLMAX]', where `YYLMAX' is a macro definition that you can + redefine in the first section if you don't like the default value + (generally 8KB). Using `%array' results in somewhat slower + scanners, but the value of `yytext' becomes immune to calls to + `input()' and `unput()', which potentially destroy its value when + `yytext' is a character pointer. The opposite of `%array' is + `%pointer', which is the default. + + You cannot use `%array' when generating C++ scanner classes (the + `-+' flag). + + - `int yyleng' holds the length of the current token. + + - `FILE *yyin' is the file which by default `flex' reads from. It + may be redefined but doing so only makes sense before scanning + begins or after an EOF has been encountered. Changing it in the + midst of scanning will have unexpected results since `flex' + buffers its input; use `yyrestart()' instead. Once scanning + terminates because an end-of-file has been seen, you can assign + `yyin' at the new input file and then call the scanner again to + continue scanning. + + - `void yyrestart( FILE *new_file )' may be called to point `yyin' + at the new input file. The switch-over to the new file is + immediate (any previously buffered-up input is lost). Note that + calling `yyrestart()' with `yyin' as an argument thus throws away + the current input buffer and continues scanning the same input + file. + + - `FILE *yyout' is the file to which `ECHO' actions are done. It + can be reassigned by the user. + + - `YY_CURRENT_BUFFER' returns a `YY_BUFFER_STATE' handle to the + current buffer. + + - `YY_START' returns an integer value corresponding to the current + start condition. You can subsequently use this value with `BEGIN' + to return to that start condition. + + +File: flex.info, Node: YACC interface, Next: Options, Prev: User variables, Up: Top + +Interfacing with `yacc' +======================= + + One of the main uses of `flex' is as a companion to the `yacc' +parser-generator. `yacc' parsers expect to call a routine named +`yylex()' to find the next input token. The routine is supposed to +return the type of the next token as well as putting any associated +value in the global `yylval'. To use `flex' with `yacc', one specifies +the `-d' option to `yacc' to instruct it to generate the file `y.tab.h' +containing definitions of all the `%tokens' appearing in the `yacc' +input. This file is then included in the `flex' scanner. For example, +if one of the tokens is "TOK_NUMBER", part of the scanner might look +like: + + %{ + #include "y.tab.h" + %} + + %% + + [0-9]+ yylval = atoi( yytext ); return TOK_NUMBER; + + +File: flex.info, Node: Options, Next: Performance, Prev: YACC interface, Up: Top + +Options +======= + + `flex' has the following options: + +`-b' + Generate backing-up information to `lex.backup'. This is a list + of scanner states which require backing up and the input + characters on which they do so. By adding rules one can remove + backing-up states. If *all* backing-up states are eliminated and + `-Cf' or `-CF' is used, the generated scanner will run faster (see + the `-p' flag). Only users who wish to squeeze every last cycle + out of their scanners need worry about this option. (See the + section on Performance Considerations below.) + +`-c' + is a do-nothing, deprecated option included for POSIX compliance. + +`-d' + makes the generated scanner run in "debug" mode. Whenever a + pattern is recognized and the global `yy_flex_debug' is non-zero + (which is the default), the scanner will write to `stderr' a line + of the form: + + --accepting rule at line 53 ("the matched text") + + The line number refers to the location of the rule in the file + defining the scanner (i.e., the file that was fed to flex). + Messages are also generated when the scanner backs up, accepts the + default rule, reaches the end of its input buffer (or encounters a + NUL; at this point, the two look the same as far as the scanner's + concerned), or reaches an end-of-file. + +`-f' + specifies "fast scanner". No table compression is done and stdio + is bypassed. The result is large but fast. This option is + equivalent to `-Cfr' (see below). + +`-h' + generates a "help" summary of `flex's' options to `stdout' and + then exits. `-?' and `--help' are synonyms for `-h'. + +`-i' + instructs `flex' to generate a *case-insensitive* scanner. The + case of letters given in the `flex' input patterns will be + ignored, and tokens in the input will be matched regardless of + case. The matched text given in `yytext' will have the preserved + case (i.e., it will not be folded). + +`-l' + turns on maximum compatibility with the original AT&T `lex' + implementation. Note that this does not mean *full* + compatibility. Use of this option costs a considerable amount of + performance, and it cannot be used with the `-+, -f, -F, -Cf', or + `-CF' options. For details on the compatibilities it provides, see + the section "Incompatibilities With Lex And POSIX" below. This + option also results in the name `YY_FLEX_LEX_COMPAT' being + #define'd in the generated scanner. + +`-n' + is another do-nothing, deprecated option included only for POSIX + compliance. + +`-p' + generates a performance report to stderr. The report consists of + comments regarding features of the `flex' input file which will + cause a serious loss of performance in the resulting scanner. If + you give the flag twice, you will also get comments regarding + features that lead to minor performance losses. + + Note that the use of `REJECT', `%option yylineno' and variable + trailing context (see the Deficiencies / Bugs section below) + entails a substantial performance penalty; use of `yymore()', the + `^' operator, and the `-I' flag entail minor performance penalties. + +`-s' + causes the "default rule" (that unmatched scanner input is echoed + to `stdout') to be suppressed. If the scanner encounters input + that does not match any of its rules, it aborts with an error. + This option is useful for finding holes in a scanner's rule set. + +`-t' + instructs `flex' to write the scanner it generates to standard + output instead of `lex.yy.c'. + +`-v' + specifies that `flex' should write to `stderr' a summary of + statistics regarding the scanner it generates. Most of the + statistics are meaningless to the casual `flex' user, but the + first line identifies the version of `flex' (same as reported by + `-V'), and the next line the flags used when generating the + scanner, including those that are on by default. + +`-w' + suppresses warning messages. + +`-B' + instructs `flex' to generate a *batch* scanner, the opposite of + *interactive* scanners generated by `-I' (see below). In general, + you use `-B' when you are *certain* that your scanner will never + be used interactively, and you want to squeeze a *little* more + performance out of it. If your goal is instead to squeeze out a + *lot* more performance, you should be using the `-Cf' or `-CF' + options (discussed below), which turn on `-B' automatically anyway. + +`-F' + specifies that the "fast" scanner table representation should be + used (and stdio bypassed). This representation is about as fast + as the full table representation `(-f)', and for some sets of + patterns will be considerably smaller (and for others, larger). + In general, if the pattern set contains both "keywords" and a + catch-all, "identifier" rule, such as in the set: + + "case" return TOK_CASE; + "switch" return TOK_SWITCH; + ... + "default" return TOK_DEFAULT; + [a-z]+ return TOK_ID; + + then you're better off using the full table representation. If + only the "identifier" rule is present and you then use a hash + table or some such to detect the keywords, you're better off using + `-F'. + + This option is equivalent to `-CFr' (see below). It cannot be + used with `-+'. + +`-I' + instructs `flex' to generate an *interactive* scanner. An + interactive scanner is one that only looks ahead to decide what + token has been matched if it absolutely must. It turns out that + always looking one extra character ahead, even if the scanner has + already seen enough text to disambiguate the current token, is a + bit faster than only looking ahead when necessary. But scanners + that always look ahead give dreadful interactive performance; for + example, when a user types a newline, it is not recognized as a + newline token until they enter *another* token, which often means + typing in another whole line. + + `Flex' scanners default to *interactive* unless you use the `-Cf' + or `-CF' table-compression options (see below). That's because if + you're looking for high-performance you should be using one of + these options, so if you didn't, `flex' assumes you'd rather trade + off a bit of run-time performance for intuitive interactive + behavior. Note also that you *cannot* use `-I' in conjunction + with `-Cf' or `-CF'. Thus, this option is not really needed; it + is on by default for all those cases in which it is allowed. + + You can force a scanner to *not* be interactive by using `-B' (see + above). + +`-L' + instructs `flex' not to generate `#line' directives. Without this + option, `flex' peppers the generated scanner with #line directives + so error messages in the actions will be correctly located with + respect to either the original `flex' input file (if the errors + are due to code in the input file), or `lex.yy.c' (if the errors + are `flex's' fault - you should report these sorts of errors to + the email address given below). + +`-T' + makes `flex' run in `trace' mode. It will generate a lot of + messages to `stderr' concerning the form of the input and the + resultant non-deterministic and deterministic finite automata. + This option is mostly for use in maintaining `flex'. + +`-V' + prints the version number to `stdout' and exits. `--version' is a + synonym for `-V'. + +`-7' + instructs `flex' to generate a 7-bit scanner, i.e., one which can + only recognized 7-bit characters in its input. The advantage of + using `-7' is that the scanner's tables can be up to half the size + of those generated using the `-8' option (see below). The + disadvantage is that such scanners often hang or crash if their + input contains an 8-bit character. + + Note, however, that unless you generate your scanner using the + `-Cf' or `-CF' table compression options, use of `-7' will save + only a small amount of table space, and make your scanner + considerably less portable. `Flex's' default behavior is to + generate an 8-bit scanner unless you use the `-Cf' or `-CF', in + which case `flex' defaults to generating 7-bit scanners unless + your site was always configured to generate 8-bit scanners (as + will often be the case with non-USA sites). You can tell whether + flex generated a 7-bit or an 8-bit scanner by inspecting the flag + summary in the `-v' output as described above. + + Note that if you use `-Cfe' or `-CFe' (those table compression + options, but also using equivalence classes as discussed see + below), flex still defaults to generating an 8-bit scanner, since + usually with these compression options full 8-bit tables are not + much more expensive than 7-bit tables. + +`-8' + instructs `flex' to generate an 8-bit scanner, i.e., one which can + recognize 8-bit characters. This flag is only needed for scanners + generated using `-Cf' or `-CF', as otherwise flex defaults to + generating an 8-bit scanner anyway. + + See the discussion of `-7' above for flex's default behavior and + the tradeoffs between 7-bit and 8-bit scanners. + +`-+' + specifies that you want flex to generate a C++ scanner class. See + the section on Generating C++ Scanners below for details. + +`-C[aefFmr]' + controls the degree of table compression and, more generally, + trade-offs between small scanners and fast scanners. + + `-Ca' ("align") instructs flex to trade off larger tables in the + generated scanner for faster performance because the elements of + the tables are better aligned for memory access and computation. + On some RISC architectures, fetching and manipulating long-words + is more efficient than with smaller-sized units such as + shortwords. This option can double the size of the tables used by + your scanner. + + `-Ce' directs `flex' to construct "equivalence classes", i.e., + sets of characters which have identical lexical properties (for + example, if the only appearance of digits in the `flex' input is + in the character class "[0-9]" then the digits '0', '1', ..., '9' + will all be put in the same equivalence class). Equivalence + classes usually give dramatic reductions in the final table/object + file sizes (typically a factor of 2-5) and are pretty cheap + performance-wise (one array look-up per character scanned). + + `-Cf' specifies that the *full* scanner tables should be generated + - `flex' should not compress the tables by taking advantages of + similar transition functions for different states. + + `-CF' specifies that the alternate fast scanner representation + (described above under the `-F' flag) should be used. This option + cannot be used with `-+'. + + `-Cm' directs `flex' to construct "meta-equivalence classes", + which are sets of equivalence classes (or characters, if + equivalence classes are not being used) that are commonly used + together. Meta-equivalence classes are often a big win when using + compressed tables, but they have a moderate performance impact + (one or two "if" tests and one array look-up per character + scanned). + + `-Cr' causes the generated scanner to *bypass* use of the standard + I/O library (stdio) for input. Instead of calling `fread()' or + `getc()', the scanner will use the `read()' system call, resulting + in a performance gain which varies from system to system, but in + general is probably negligible unless you are also using `-Cf' or + `-CF'. Using `-Cr' can cause strange behavior if, for example, + you read from `yyin' using stdio prior to calling the scanner + (because the scanner will miss whatever text your previous reads + left in the stdio input buffer). + + `-Cr' has no effect if you define `YY_INPUT' (see The Generated + Scanner above). + + A lone `-C' specifies that the scanner tables should be compressed + but neither equivalence classes nor meta-equivalence classes + should be used. + + The options `-Cf' or `-CF' and `-Cm' do not make sense together - + there is no opportunity for meta-equivalence classes if the table + is not being compressed. Otherwise the options may be freely + mixed, and are cumulative. + + The default setting is `-Cem', which specifies that `flex' should + generate equivalence classes and meta-equivalence classes. This + setting provides the highest degree of table compression. You can + trade off faster-executing scanners at the cost of larger tables + with the following generally being true: + + slowest & smallest + -Cem + -Cm + -Ce + -C + -C{f,F}e + -C{f,F} + -C{f,F}a + fastest & largest + + Note that scanners with the smallest tables are usually generated + and compiled the quickest, so during development you will usually + want to use the default, maximal compression. + + `-Cfe' is often a good compromise between speed and size for + production scanners. + +`-ooutput' + directs flex to write the scanner to the file `out-' `put' instead + of `lex.yy.c'. If you combine `-o' with the `-t' option, then the + scanner is written to `stdout' but its `#line' directives (see the + `-L' option above) refer to the file `output'. + +`-Pprefix' + changes the default `yy' prefix used by `flex' for all + globally-visible variable and function names to instead be PREFIX. + For example, `-Pfoo' changes the name of `yytext' to `footext'. + It also changes the name of the default output file from + `lex.yy.c' to `lex.foo.c'. Here are all of the names affected: + + yy_create_buffer + yy_delete_buffer + yy_flex_debug + yy_init_buffer + yy_flush_buffer + yy_load_buffer_state + yy_switch_to_buffer + yyin + yyleng + yylex + yylineno + yyout + yyrestart + yytext + yywrap + + (If you are using a C++ scanner, then only `yywrap' and + `yyFlexLexer' are affected.) Within your scanner itself, you can + still refer to the global variables and functions using either + version of their name; but externally, they have the modified name. + + This option lets you easily link together multiple `flex' programs + into the same executable. Note, though, that using this option + also renames `yywrap()', so you now *must* either provide your own + (appropriately-named) version of the routine for your scanner, or + use `%option noyywrap', as linking with `-lfl' no longer provides + one for you by default. + +`-Sskeleton_file' + overrides the default skeleton file from which `flex' constructs + its scanners. You'll never need this option unless you are doing + `flex' maintenance or development. + + `flex' also provides a mechanism for controlling options within the +scanner specification itself, rather than from the flex command-line. +This is done by including `%option' directives in the first section of +the scanner specification. You can specify multiple options with a +single `%option' directive, and multiple directives in the first +section of your flex input file. Most options are given simply as +names, optionally preceded by the word "no" (with no intervening +whitespace) to negate their meaning. A number are equivalent to flex +flags or their negation: + + 7bit -7 option + 8bit -8 option + align -Ca option + backup -b option + batch -B option + c++ -+ option + + caseful or + case-sensitive opposite of -i (default) + + case-insensitive or + caseless -i option + + debug -d option + default opposite of -s option + ecs -Ce option + fast -F option + full -f option + interactive -I option + lex-compat -l option + meta-ecs -Cm option + perf-report -p option + read -Cr option + stdout -t option + verbose -v option + warn opposite of -w option + (use "%option nowarn" for -w) + + array equivalent to "%array" + pointer equivalent to "%pointer" (default) + + Some `%option's' provide features otherwise not available: + +`always-interactive' + instructs flex to generate a scanner which always considers its + input "interactive". Normally, on each new input file the scanner + calls `isatty()' in an attempt to determine whether the scanner's + input source is interactive and thus should be read a character at + a time. When this option is used, however, then no such call is + made. + +`main' + directs flex to provide a default `main()' program for the + scanner, which simply calls `yylex()'. This option implies + `noyywrap' (see below). + +`never-interactive' + instructs flex to generate a scanner which never considers its + input "interactive" (again, no call made to `isatty())'. This is + the opposite of `always-' *interactive*. + +`stack' + enables the use of start condition stacks (see Start Conditions + above). + +`stdinit' + if unset (i.e., `%option nostdinit') initializes `yyin' and + `yyout' to nil `FILE' pointers, instead of `stdin' and `stdout'. + +`yylineno' + directs `flex' to generate a scanner that maintains the number of + the current line read from its input in the global variable + `yylineno'. This option is implied by `%option lex-compat'. + +`yywrap' + if unset (i.e., `%option noyywrap'), makes the scanner not call + `yywrap()' upon an end-of-file, but simply assume that there are + no more files to scan (until the user points `yyin' at a new file + and calls `yylex()' again). + + `flex' scans your rule actions to determine whether you use the +`REJECT' or `yymore()' features. The `reject' and `yymore' options are +available to override its decision as to whether you use the options, +either by setting them (e.g., `%option reject') to indicate the feature +is indeed used, or unsetting them to indicate it actually is not used +(e.g., `%option noyymore'). + + Three options take string-delimited values, offset with '=': + + %option outfile="ABC" + +is equivalent to `-oABC', and + + %option prefix="XYZ" + +is equivalent to `-PXYZ'. + + Finally, + + %option yyclass="foo" + +only applies when generating a C++ scanner (`-+' option). It informs +`flex' that you have derived `foo' as a subclass of `yyFlexLexer' so +`flex' will place your actions in the member function `foo::yylex()' +instead of `yyFlexLexer::yylex()'. It also generates a +`yyFlexLexer::yylex()' member function that emits a run-time error (by +invoking `yyFlexLexer::LexerError()') if called. See Generating C++ +Scanners, below, for additional information. + + A number of options are available for lint purists who want to +suppress the appearance of unneeded routines in the generated scanner. +Each of the following, if unset, results in the corresponding routine +not appearing in the generated scanner: + + input, unput + yy_push_state, yy_pop_state, yy_top_state + yy_scan_buffer, yy_scan_bytes, yy_scan_string + +(though `yy_push_state()' and friends won't appear anyway unless you +use `%option stack'). + + +File: flex.info, Node: Performance, Next: C++, Prev: Options, Up: Top + +Performance considerations +========================== + + The main design goal of `flex' is that it generate high-performance +scanners. It has been optimized for dealing well with large sets of +rules. Aside from the effects on scanner speed of the table +compression `-C' options outlined above, there are a number of +options/actions which degrade performance. These are, from most +expensive to least: + + REJECT + %option yylineno + arbitrary trailing context + + pattern sets that require backing up + %array + %option interactive + %option always-interactive + + '^' beginning-of-line operator + yymore() + + with the first three all being quite expensive and the last two +being quite cheap. Note also that `unput()' is implemented as a +routine call that potentially does quite a bit of work, while +`yyless()' is a quite-cheap macro; so if just putting back some excess +text you scanned, use `yyless()'. + + `REJECT' should be avoided at all costs when performance is +important. It is a particularly expensive option. + + Getting rid of backing up is messy and often may be an enormous +amount of work for a complicated scanner. In principal, one begins by +using the `-b' flag to generate a `lex.backup' file. For example, on +the input + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + +the file looks like: + + State #6 is non-accepting - + associated rule line numbers: + 2 3 + out-transitions: [ o ] + jam-transitions: EOF [ \001-n p-\177 ] + + State #8 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ a ] + jam-transitions: EOF [ \001-` b-\177 ] + + State #9 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ r ] + jam-transitions: EOF [ \001-q s-\177 ] + + Compressed tables always back up. + + The first few lines tell us that there's a scanner state in which it +can make a transition on an 'o' but not on any other character, and +that in that state the currently scanned text does not match any rule. +The state occurs when trying to match the rules found at lines 2 and 3 +in the input file. If the scanner is in that state and then reads +something other than an 'o', it will have to back up to find a rule +which is matched. With a bit of head-scratching one can see that this +must be the state it's in when it has seen "fo". When this has +happened, if anything other than another 'o' is seen, the scanner will +have to back up to simply match the 'f' (by the default rule). + + The comment regarding State #8 indicates there's a problem when +"foob" has been scanned. Indeed, on any character other than an 'a', +the scanner will have to back up to accept "foo". Similarly, the +comment for State #9 concerns when "fooba" has been scanned and an 'r' +does not follow. + + The final comment reminds us that there's no point going to all the +trouble of removing backing up from the rules unless we're using `-Cf' +or `-CF', since there's no performance gain doing so with compressed +scanners. + + The way to remove the backing up is to add "error" rules: + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + fooba | + foob | + fo { + /* false alarm, not really a keyword */ + return TOK_ID; + } + + Eliminating backing up among a list of keywords can also be done +using a "catch-all" rule: + + %% + foo return TOK_KEYWORD; + foobar return TOK_KEYWORD; + + [a-z]+ return TOK_ID; + + This is usually the best solution when appropriate. + + Backing up messages tend to cascade. With a complicated set of +rules it's not uncommon to get hundreds of messages. If one can +decipher them, though, it often only takes a dozen or so rules to +eliminate the backing up (though it's easy to make a mistake and have +an error rule accidentally match a valid token. A possible future +`flex' feature will be to automatically add rules to eliminate backing +up). + + It's important to keep in mind that you gain the benefits of +eliminating backing up only if you eliminate *every* instance of +backing up. Leaving just one means you gain nothing. + + VARIABLE trailing context (where both the leading and trailing parts +do not have a fixed length) entails almost the same performance loss as +`REJECT' (i.e., substantial). So when possible a rule like: + + %% + mouse|rat/(cat|dog) run(); + +is better written: + + %% + mouse/cat|dog run(); + rat/cat|dog run(); + +or as + + %% + mouse|rat/cat run(); + mouse|rat/dog run(); + + Note that here the special '|' action does *not* provide any +savings, and can even make things worse (see Deficiencies / Bugs below). + + Another area where the user can increase a scanner's performance +(and one that's easier to implement) arises from the fact that the +longer the tokens matched, the faster the scanner will run. This is +because with long tokens the processing of most input characters takes +place in the (short) inner scanning loop, and does not often have to go +through the additional work of setting up the scanning environment +(e.g., `yytext') for the action. Recall the scanner for C comments: + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* + <comment>"*"+[^*/\n]* + <comment>\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + This could be sped up by writing it as: + + %x comment + %% + int line_num = 1; + + "/*" BEGIN(comment); + + <comment>[^*\n]* + <comment>[^*\n]*\n ++line_num; + <comment>"*"+[^*/\n]* + <comment>"*"+[^*/\n]*\n ++line_num; + <comment>"*"+"/" BEGIN(INITIAL); + + Now instead of each newline requiring the processing of another +action, recognizing the newlines is "distributed" over the other rules +to keep the matched text as long as possible. Note that *adding* rules +does *not* slow down the scanner! The speed of the scanner is +independent of the number of rules or (modulo the considerations given +at the beginning of this section) how complicated the rules are with +regard to operators such as '*' and '|'. + + A final example in speeding up a scanner: suppose you want to scan +through a file containing identifiers and keywords, one per line and +with no other extraneous characters, and recognize all the keywords. A +natural first approach is: + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + .|\n /* it's not a keyword */ + + To eliminate the back-tracking, introduce a catch-all rule: + + %% + asm | + auto | + break | + ... etc ... + volatile | + while /* it's a keyword */ + + [a-z]+ | + .|\n /* it's not a keyword */ + + Now, if it's guaranteed that there's exactly one word per line, then +we can reduce the total number of matches by a half by merging in the +recognition of newlines with that of the other tokens: + + %% + asm\n | + auto\n | + break\n | + ... etc ... + volatile\n | + while\n /* it's a keyword */ + + [a-z]+\n | + .|\n /* it's not a keyword */ + + One has to be careful here, as we have now reintroduced backing up +into the scanner. In particular, while *we* know that there will never +be any characters in the input stream other than letters or newlines, +`flex' can't figure this out, and it will plan for possibly needing to +back up when it has scanned a token like "auto" and then the next +character is something other than a newline or a letter. Previously it +would then just match the "auto" rule and be done, but now it has no +"auto" rule, only a "auto\n" rule. To eliminate the possibility of +backing up, we could either duplicate all rules but without final +newlines, or, since we never expect to encounter such an input and +therefore don't how it's classified, we can introduce one more +catch-all rule, this one which doesn't include a newline: + + %% + asm\n | + auto\n | + break\n | + ... etc ... + volatile\n | + while\n /* it's a keyword */ + + [a-z]+\n | + [a-z]+ | + .|\n /* it's not a keyword */ + + Compiled with `-Cf', this is about as fast as one can get a `flex' +scanner to go for this particular problem. + + A final note: `flex' is slow when matching NUL's, particularly when +a token contains multiple NUL's. It's best to write rules which match +*short* amounts of text if it's anticipated that the text will often +include NUL's. + + Another final note regarding performance: as mentioned above in the +section How the Input is Matched, dynamically resizing `yytext' to +accommodate huge tokens is a slow process because it presently requires +that the (huge) token be rescanned from the beginning. Thus if +performance is vital, you should attempt to match "large" quantities of +text but not "huge" quantities, where the cutoff between the two is at +about 8K characters/token. + + +File: flex.info, Node: C++, Next: Incompatibilities, Prev: Performance, Up: Top + +Generating C++ scanners +======================= + + `flex' provides two different ways to generate scanners for use with +C++. The first way is to simply compile a scanner generated by `flex' +using a C++ compiler instead of a C compiler. You should not encounter +any compilations errors (please report any you find to the email address +given in the Author section below). You can then use C++ code in your +rule actions instead of C code. Note that the default input source for +your scanner remains `yyin', and default echoing is still done to +`yyout'. Both of these remain `FILE *' variables and not C++ `streams'. + + You can also use `flex' to generate a C++ scanner class, using the +`-+' option, (or, equivalently, `%option c++'), which is automatically +specified if the name of the flex executable ends in a `+', such as +`flex++'. When using this option, flex defaults to generating the +scanner to the file `lex.yy.cc' instead of `lex.yy.c'. The generated +scanner includes the header file `FlexLexer.h', which defines the +interface to two C++ classes. + + The first class, `FlexLexer', provides an abstract base class +defining the general scanner class interface. It provides the +following member functions: + +`const char* YYText()' + returns the text of the most recently matched token, the + equivalent of `yytext'. + +`int YYLeng()' + returns the length of the most recently matched token, the + equivalent of `yyleng'. + +`int lineno() const' + returns the current input line number (see `%option yylineno'), or + 1 if `%option yylineno' was not used. + +`void set_debug( int flag )' + sets the debugging flag for the scanner, equivalent to assigning to + `yy_flex_debug' (see the Options section above). Note that you + must build the scanner using `%option debug' to include debugging + information in it. + +`int debug() const' + returns the current setting of the debugging flag. + + Also provided are member functions equivalent to +`yy_switch_to_buffer(), yy_create_buffer()' (though the first argument +is an `istream*' object pointer and not a `FILE*', `yy_flush_buffer()', +`yy_delete_buffer()', and `yyrestart()' (again, the first argument is a +`istream*' object pointer). + + The second class defined in `FlexLexer.h' is `yyFlexLexer', which is +derived from `FlexLexer'. It defines the following additional member +functions: + +`yyFlexLexer( istream* arg_yyin = 0, ostream* arg_yyout = 0 )' + constructs a `yyFlexLexer' object using the given streams for + input and output. If not specified, the streams default to `cin' + and `cout', respectively. + +`virtual int yylex()' + performs the same role is `yylex()' does for ordinary flex + scanners: it scans the input stream, consuming tokens, until a + rule's action returns a value. If you derive a subclass S from + `yyFlexLexer' and want to access the member functions and + variables of S inside `yylex()', then you need to use `%option + yyclass="S"' to inform `flex' that you will be using that subclass + instead of `yyFlexLexer'. In this case, rather than generating + `yyFlexLexer::yylex()', `flex' generates `S::yylex()' (and also + generates a dummy `yyFlexLexer::yylex()' that calls + `yyFlexLexer::LexerError()' if called). + +`virtual void switch_streams(istream* new_in = 0, ostream* new_out = 0)' + reassigns `yyin' to `new_in' (if non-nil) and `yyout' to `new_out' + (ditto), deleting the previous input buffer if `yyin' is + reassigned. + +`int yylex( istream* new_in = 0, ostream* new_out = 0 )' + first switches the input streams via `switch_streams( new_in, + new_out )' and then returns the value of `yylex()'. + + In addition, `yyFlexLexer' defines the following protected virtual +functions which you can redefine in derived classes to tailor the +scanner: + +`virtual int LexerInput( char* buf, int max_size )' + reads up to `max_size' characters into BUF and returns the number + of characters read. To indicate end-of-input, return 0 + characters. Note that "interactive" scanners (see the `-B' and + `-I' flags) define the macro `YY_INTERACTIVE'. If you redefine + `LexerInput()' and need to take different actions depending on + whether or not the scanner might be scanning an interactive input + source, you can test for the presence of this name via `#ifdef'. + +`virtual void LexerOutput( const char* buf, int size )' + writes out SIZE characters from the buffer BUF, which, while + NUL-terminated, may also contain "internal" NUL's if the scanner's + rules can match text with NUL's in them. + +`virtual void LexerError( const char* msg )' + reports a fatal error message. The default version of this + function writes the message to the stream `cerr' and exits. + + Note that a `yyFlexLexer' object contains its *entire* scanning +state. Thus you can use such objects to create reentrant scanners. +You can instantiate multiple instances of the same `yyFlexLexer' class, +and you can also combine multiple C++ scanner classes together in the +same program using the `-P' option discussed above. Finally, note that +the `%array' feature is not available to C++ scanner classes; you must +use `%pointer' (the default). + + Here is an example of a simple C++ scanner: + + // An example of using the flex C++ scanner class. + + %{ + int mylineno = 0; + %} + + string \"[^\n"]+\" + + ws [ \t]+ + + alpha [A-Za-z] + dig [0-9] + name ({alpha}|{dig}|\$)({alpha}|{dig}|[_.\-/$])* + num1 [-+]?{dig}+\.?([eE][-+]?{dig}+)? + num2 [-+]?{dig}*\.{dig}+([eE][-+]?{dig}+)? + number {num1}|{num2} + + %% + + {ws} /* skip blanks and tabs */ + + "/*" { + int c; + + while((c = yyinput()) != 0) + { + if(c == '\n') + ++mylineno; + + else if(c == '*') + { + if((c = yyinput()) == '/') + break; + else + unput(c); + } + } + } + + {number} cout << "number " << YYText() << '\n'; + + \n mylineno++; + + {name} cout << "name " << YYText() << '\n'; + + {string} cout << "string " << YYText() << '\n'; + + %% + + Version 2.5 December 1994 44 + + int main( int /* argc */, char** /* argv */ ) + { + FlexLexer* lexer = new yyFlexLexer; + while(lexer->yylex() != 0) + ; + return 0; + } + + If you want to create multiple (different) lexer classes, you use +the `-P' flag (or the `prefix=' option) to rename each `yyFlexLexer' to +some other `xxFlexLexer'. You then can include `<FlexLexer.h>' in your +other sources once per lexer class, first renaming `yyFlexLexer' as +follows: + + #undef yyFlexLexer + #define yyFlexLexer xxFlexLexer + #include <FlexLexer.h> + + #undef yyFlexLexer + #define yyFlexLexer zzFlexLexer + #include <FlexLexer.h> + + if, for example, you used `%option prefix="xx"' for one of your +scanners and `%option prefix="zz"' for the other. + + IMPORTANT: the present form of the scanning class is *experimental* +and may change considerably between major releases. + + +File: flex.info, Node: Incompatibilities, Next: Diagnostics, Prev: C++, Up: Top + +Incompatibilities with `lex' and POSIX +====================================== + + `flex' is a rewrite of the AT&T Unix `lex' tool (the two +implementations do not share any code, though), with some extensions +and incompatibilities, both of which are of concern to those who wish +to write scanners acceptable to either implementation. Flex is fully +compliant with the POSIX `lex' specification, except that when using +`%pointer' (the default), a call to `unput()' destroys the contents of +`yytext', which is counter to the POSIX specification. + + In this section we discuss all of the known areas of incompatibility +between flex, AT&T lex, and the POSIX specification. + + `flex's' `-l' option turns on maximum compatibility with the +original AT&T `lex' implementation, at the cost of a major loss in the +generated scanner's performance. We note below which incompatibilities +can be overcome using the `-l' option. + + `flex' is fully compatible with `lex' with the following exceptions: + + - The undocumented `lex' scanner internal variable `yylineno' is not + supported unless `-l' or `%option yylineno' is used. `yylineno' + should be maintained on a per-buffer basis, rather than a + per-scanner (single global variable) basis. `yylineno' is not + part of the POSIX specification. + + - The `input()' routine is not redefinable, though it may be called + to read characters following whatever has been matched by a rule. + If `input()' encounters an end-of-file the normal `yywrap()' + processing is done. A "real" end-of-file is returned by `input()' + as `EOF'. + + Input is instead controlled by defining the `YY_INPUT' macro. + + The `flex' restriction that `input()' cannot be redefined is in + accordance with the POSIX specification, which simply does not + specify any way of controlling the scanner's input other than by + making an initial assignment to `yyin'. + + - The `unput()' routine is not redefinable. This restriction is in + accordance with POSIX. + + - `flex' scanners are not as reentrant as `lex' scanners. In + particular, if you have an interactive scanner and an interrupt + handler which long-jumps out of the scanner, and the scanner is + subsequently called again, you may get the following message: + + fatal flex scanner internal error--end of buffer missed + + To reenter the scanner, first use + + yyrestart( yyin ); + + Note that this call will throw away any buffered input; usually + this isn't a problem with an interactive scanner. + + Also note that flex C++ scanner classes *are* reentrant, so if + using C++ is an option for you, you should use them instead. See + "Generating C++ Scanners" above for details. + + - `output()' is not supported. Output from the `ECHO' macro is done + to the file-pointer `yyout' (default `stdout'). + + `output()' is not part of the POSIX specification. + + - `lex' does not support exclusive start conditions (%x), though + they are in the POSIX specification. + + - When definitions are expanded, `flex' encloses them in + parentheses. With lex, the following: + + NAME [A-Z][A-Z0-9]* + %% + foo{NAME}? printf( "Found it\n" ); + %% + + will not match the string "foo" because when the macro is expanded + the rule is equivalent to "foo[A-Z][A-Z0-9]*?" and the precedence + is such that the '?' is associated with "[A-Z0-9]*". With `flex', + the rule will be expanded to "foo([A-Z][A-Z0-9]*)?" and so the + string "foo" will match. + + Note that if the definition begins with `^' or ends with `$' then + it is *not* expanded with parentheses, to allow these operators to + appear in definitions without losing their special meanings. But + the `<s>, /', and `<<EOF>>' operators cannot be used in a `flex' + definition. + + Using `-l' results in the `lex' behavior of no parentheses around + the definition. + + The POSIX specification is that the definition be enclosed in + parentheses. + + - Some implementations of `lex' allow a rule's action to begin on a + separate line, if the rule's pattern has trailing whitespace: + + %% + foo|bar<space here> + { foobar_action(); } + + `flex' does not support this feature. + + - The `lex' `%r' (generate a Ratfor scanner) option is not + supported. It is not part of the POSIX specification. + + - After a call to `unput()', `yytext' is undefined until the next + token is matched, unless the scanner was built using `%array'. + This is not the case with `lex' or the POSIX specification. The + `-l' option does away with this incompatibility. + + - The precedence of the `{}' (numeric range) operator is different. + `lex' interprets "abc{1,3}" as "match one, two, or three + occurrences of 'abc'", whereas `flex' interprets it as "match 'ab' + followed by one, two, or three occurrences of 'c'". The latter is + in agreement with the POSIX specification. + + - The precedence of the `^' operator is different. `lex' interprets + "^foo|bar" as "match either 'foo' at the beginning of a line, or + 'bar' anywhere", whereas `flex' interprets it as "match either + 'foo' or 'bar' if they come at the beginning of a line". The + latter is in agreement with the POSIX specification. + + - The special table-size declarations such as `%a' supported by + `lex' are not required by `flex' scanners; `flex' ignores them. + + - The name FLEX_SCANNER is #define'd so scanners may be written for + use with either `flex' or `lex'. Scanners also include + `YY_FLEX_MAJOR_VERSION' and `YY_FLEX_MINOR_VERSION' indicating + which version of `flex' generated the scanner (for example, for the + 2.5 release, these defines would be 2 and 5 respectively). + + The following `flex' features are not included in `lex' or the POSIX +specification: + + C++ scanners + %option + start condition scopes + start condition stacks + interactive/non-interactive scanners + yy_scan_string() and friends + yyterminate() + yy_set_interactive() + yy_set_bol() + YY_AT_BOL() + <<EOF>> + <*> + YY_DECL + YY_START + YY_USER_ACTION + YY_USER_INIT + #line directives + %{}'s around actions + multiple actions on a line + +plus almost all of the flex flags. The last feature in the list refers +to the fact that with `flex' you can put multiple actions on the same +line, separated with semicolons, while with `lex', the following + + foo handle_foo(); ++num_foos_seen; + +is (rather surprisingly) truncated to + + foo handle_foo(); + + `flex' does not truncate the action. Actions that are not enclosed +in braces are simply terminated at the end of the line. + + +File: flex.info, Node: Diagnostics, Next: Files, Prev: Incompatibilities, Up: Top + +Diagnostics +=========== + +`warning, rule cannot be matched' + indicates that the given rule cannot be matched because it follows + other rules that will always match the same text as it. For + example, in the following "foo" cannot be matched because it comes + after an identifier "catch-all" rule: + + [a-z]+ got_identifier(); + foo got_foo(); + + Using `REJECT' in a scanner suppresses this warning. + +`warning, -s option given but default rule can be matched' + means that it is possible (perhaps only in a particular start + condition) that the default rule (match any single character) is + the only one that will match a particular input. Since `-s' was + given, presumably this is not intended. + +`reject_used_but_not_detected undefined' +`yymore_used_but_not_detected undefined' + These errors can occur at compile time. They indicate that the + scanner uses `REJECT' or `yymore()' but that `flex' failed to + notice the fact, meaning that `flex' scanned the first two sections + looking for occurrences of these actions and failed to find any, + but somehow you snuck some in (via a #include file, for example). + Use `%option reject' or `%option yymore' to indicate to flex that + you really do use these features. + +`flex scanner jammed' + a scanner compiled with `-s' has encountered an input string which + wasn't matched by any of its rules. This error can also occur due + to internal problems. + +`token too large, exceeds YYLMAX' + your scanner uses `%array' and one of its rules matched a string + longer than the `YYL-' `MAX' constant (8K bytes by default). You + can increase the value by #define'ing `YYLMAX' in the definitions + section of your `flex' input. + +`scanner requires -8 flag to use the character 'X'' + Your scanner specification includes recognizing the 8-bit + character X and you did not specify the -8 flag, and your scanner + defaulted to 7-bit because you used the `-Cf' or `-CF' table + compression options. See the discussion of the `-7' flag for + details. + +`flex scanner push-back overflow' + you used `unput()' to push back so much text that the scanner's + buffer could not hold both the pushed-back text and the current + token in `yytext'. Ideally the scanner should dynamically resize + the buffer in this case, but at present it does not. + +`input buffer overflow, can't enlarge buffer because scanner uses REJECT' + the scanner was working on matching an extremely large token and + needed to expand the input buffer. This doesn't work with + scanners that use `REJECT'. + +`fatal flex scanner internal error--end of buffer missed' + This can occur in an scanner which is reentered after a long-jump + has jumped out (or over) the scanner's activation frame. Before + reentering the scanner, use: + + yyrestart( yyin ); + + or, as noted above, switch to using the C++ scanner class. + +`too many start conditions in <> construct!' + you listed more start conditions in a <> construct than exist (so + you must have listed at least one of them twice). + + +File: flex.info, Node: Files, Next: Deficiencies, Prev: Diagnostics, Up: Top + +Files +===== + +`-lfl' + library with which scanners must be linked. + +`lex.yy.c' + generated scanner (called `lexyy.c' on some systems). + +`lex.yy.cc' + generated C++ scanner class, when using `-+'. + +`<FlexLexer.h>' + header file defining the C++ scanner base class, `FlexLexer', and + its derived class, `yyFlexLexer'. + +`flex.skl' + skeleton scanner. This file is only used when building flex, not + when flex executes. + +`lex.backup' + backing-up information for `-b' flag (called `lex.bck' on some + systems). + + +File: flex.info, Node: Deficiencies, Next: See also, Prev: Files, Up: Top + +Deficiencies / Bugs +=================== + + Some trailing context patterns cannot be properly matched and +generate warning messages ("dangerous trailing context"). These are +patterns where the ending of the first part of the rule matches the +beginning of the second part, such as "zx*/xy*", where the 'x*' matches +the 'x' at the beginning of the trailing context. (Note that the POSIX +draft states that the text matched by such patterns is undefined.) + + For some trailing context rules, parts which are actually +fixed-length are not recognized as such, leading to the abovementioned +performance loss. In particular, parts using '|' or {n} (such as +"foo{3}") are always considered variable-length. + + Combining trailing context with the special '|' action can result in +*fixed* trailing context being turned into the more expensive VARIABLE +trailing context. For example, in the following: + + %% + abc | + xyz/def + + Use of `unput()' invalidates yytext and yyleng, unless the `%array' +directive or the `-l' option has been used. + + Pattern-matching of NUL's is substantially slower than matching +other characters. + + Dynamic resizing of the input buffer is slow, as it entails +rescanning all the text matched so far by the current (generally huge) +token. + + Due to both buffering of input and read-ahead, you cannot intermix +calls to <stdio.h> routines, such as, for example, `getchar()', with +`flex' rules and expect it to work. Call `input()' instead. + + The total table entries listed by the `-v' flag excludes the number +of table entries needed to determine what rule has been matched. The +number of entries is equal to the number of DFA states if the scanner +does not use `REJECT', and somewhat greater than the number of states +if it does. + + `REJECT' cannot be used with the `-f' or `-F' options. + + The `flex' internal algorithms need documentation. + + +File: flex.info, Node: See also, Next: Author, Prev: Deficiencies, Up: Top + +See also +======== + + `lex'(1), `yacc'(1), `sed'(1), `awk'(1). + + John Levine, Tony Mason, and Doug Brown: Lex & Yacc; O'Reilly and +Associates. Be sure to get the 2nd edition. + + M. E. Lesk and E. Schmidt, LEX - Lexical Analyzer Generator. + + Alfred Aho, Ravi Sethi and Jeffrey Ullman: Compilers: Principles, +Techniques and Tools; Addison-Wesley (1986). Describes the +pattern-matching techniques used by `flex' (deterministic finite +automata). + + +File: flex.info, Node: Author, Prev: See also, Up: Top + +Author +====== + + Vern Paxson, with the help of many ideas and much inspiration from +Van Jacobson. Original version by Jef Poskanzer. The fast table +representation is a partial implementation of a design done by Van +Jacobson. The implementation was done by Kevin Gong and Vern Paxson. + + Thanks to the many `flex' beta-testers, feedbackers, and +contributors, especially Francois Pinard, Casey Leedom, Stan Adermann, +Terry Allen, David Barker-Plummer, John Basrai, Nelson H.F. Beebe, +`benson@odi.com', Karl Berry, Peter A. Bigot, Simon Blanchard, Keith +Bostic, Frederic Brehm, Ian Brockbank, Kin Cho, Nick Christopher, Brian +Clapper, J.T. Conklin, Jason Coughlin, Bill Cox, Nick Cropper, Dave +Curtis, Scott David Daniels, Chris G. Demetriou, Theo Deraadt, Mike +Donahue, Chuck Doucette, Tom Epperly, Leo Eskin, Chris Faylor, Chris +Flatters, Jon Forrest, Joe Gayda, Kaveh R. Ghazi, Eric Goldman, +Christopher M. Gould, Ulrich Grepel, Peer Griebel, Jan Hajic, Charles +Hemphill, NORO Hideo, Jarkko Hietaniemi, Scott Hofmann, Jeff Honig, +Dana Hudes, Eric Hughes, John Interrante, Ceriel Jacobs, Michal +Jaegermann, Sakari Jalovaara, Jeffrey R. Jones, Henry Juengst, Klaus +Kaempf, Jonathan I. Kamens, Terrence O Kane, Amir Katz, +`ken@ken.hilco.com', Kevin B. Kenny, Steve Kirsch, Winfried Koenig, +Marq Kole, Ronald Lamprecht, Greg Lee, Rohan Lenard, Craig Leres, John +Levine, Steve Liddle, Mike Long, Mohamed el Lozy, Brian Madsen, Malte, +Joe Marshall, Bengt Martensson, Chris Metcalf, Luke Mewburn, Jim +Meyering, R. Alexander Milowski, Erik Naggum, G.T. Nicol, Landon Noll, +James Nordby, Marc Nozell, Richard Ohnemus, Karsten Pahnke, Sven Panne, +Roland Pesch, Walter Pelissero, Gaumond Pierre, Esmond Pitt, Jef +Poskanzer, Joe Rahmeh, Jarmo Raiha, Frederic Raimbault, Pat Rankin, +Rick Richardson, Kevin Rodgers, Kai Uwe Rommel, Jim Roskind, Alberto +Santini, Andreas Scherer, Darrell Schiebel, Raf Schietekat, Doug +Schmidt, Philippe Schnoebelen, Andreas Schwab, Alex Siegel, Eckehard +Stolz, Jan-Erik Strvmquist, Mike Stump, Paul Stuart, Dave Tallman, Ian +Lance Taylor, Chris Thewalt, Richard M. Timoney, Jodi Tsai, Paul +Tuinenga, Gary Weik, Frank Whaley, Gerhard Wilhelms, Kent Williams, Ken +Yap, Ron Zellar, Nathan Zelle, David Zuhn, and those whose names have +slipped my marginal mail-archiving skills but whose contributions are +appreciated all the same. + + Thanks to Keith Bostic, Jon Forrest, Noah Friedman, John Gilmore, +Craig Leres, John Levine, Bob Mulcahy, G.T. Nicol, Francois Pinard, +Rich Salz, and Richard Stallman for help with various distribution +headaches. + + Thanks to Esmond Pitt and Earle Horton for 8-bit character support; +to Benson Margulies and Fred Burke for C++ support; to Kent Williams +and Tom Epperly for C++ class support; to Ove Ewerlid for support of +NUL's; and to Eric Hughes for support of multiple buffers. + + This work was primarily done when I was with the Real Time Systems +Group at the Lawrence Berkeley Laboratory in Berkeley, CA. Many thanks +to all there for the support I received. + + Send comments to `vern@ee.lbl.gov'. + + + +Tag Table: +Node: Top1430 +Node: Name2808 +Node: Synopsis2933 +Node: Overview3145 +Node: Description4986 +Node: Examples5748 +Node: Format8896 +Node: Patterns11637 +Node: Matching18138 +Node: Actions21438 +Node: Generated scanner30560 +Node: Start conditions34988 +Node: Multiple buffers45069 +Node: End-of-file rules50975 +Node: Miscellaneous52508 +Node: User variables55279 +Node: YACC interface57651 +Node: Options58542 +Node: Performance78234 +Node: C++87532 +Node: Incompatibilities94993 +Node: Diagnostics101853 +Node: Files105094 +Node: Deficiencies105715 +Node: See also107684 +Node: Author108216 + +End Tag Table diff --git a/MISC/texinfo/flex.texi b/MISC/texinfo/flex.texi new file mode 100644 index 0000000..23280b1 --- /dev/null +++ b/MISC/texinfo/flex.texi @@ -0,0 +1,3448 @@ +\input texinfo +@c %**start of header +@setfilename flex.info +@settitle Flex - a scanner generator +@c @finalout +@c @setchapternewpage odd +@c %**end of header + +@set EDITION 2.5 +@set UPDATED March 1995 +@set VERSION 2.5 + +@c FIXME - Reread a printed copy with a red pen and patience. +@c FIXME - Modify all "See ..." references and replace with @xref's. + +@ifinfo +@format +START-INFO-DIR-ENTRY +* Flex: (flex). A fast scanner generator. +END-INFO-DIR-ENTRY +@end format +@end ifinfo + +@c Define new indices for commands, filenames, and options. +@c @defcodeindex cm +@c @defcodeindex fl +@c @defcodeindex op + +@c Put everything in one index (arbitrarily chosen to be the concept index). +@c @syncodeindex cm cp +@c @syncodeindex fl cp +@syncodeindex fn cp +@syncodeindex ky cp +@c @syncodeindex op cp +@syncodeindex pg cp +@syncodeindex vr cp + +@ifinfo +This file documents Flex. + +Copyright (c) 1990 The Regents of the University of California. +All rights reserved. + +This code is derived from software contributed to Berkeley by +Vern Paxson. + +The United States Government has rights in this work pursuant +to contract no. DE-AC03-76SF00098 between the United States +Department of Energy and the University of California. + +Redistribution and use in source and binary forms with or without +modification are permitted provided that: (1) source distributions +retain this entire copyright notice and comment, and (2) +distributions including binaries display the following +acknowledgement: ``This product includes software developed by the +University of California, Berkeley and its contributors'' in the +documentation or other materials provided with the distribution and +in all advertising materials mentioning features or use of this +software. Neither the name of the University nor the names of its +contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. + +@ignore +Permission is granted to process this file through TeX and print the +results, provided the printed document carries copying permission +notice identical to this one except for the removal of this paragraph +(this paragraph not being relevant to the printed manual). + +@end ignore +@end ifinfo + +@titlepage +@title Flex, version @value{VERSION} +@subtitle A fast scanner generator +@subtitle Edition @value{EDITION}, @value{UPDATED} +@author Vern Paxson + +@page +@vskip 0pt plus 1filll +Copyright @copyright{} 1990 The Regents of the University of California. +All rights reserved. + +This code is derived from software contributed to Berkeley by +Vern Paxson. + +The United States Government has rights in this work pursuant +to contract no. DE-AC03-76SF00098 between the United States +Department of Energy and the University of California. + +Redistribution and use in source and binary forms with or without +modification are permitted provided that: (1) source distributions +retain this entire copyright notice and comment, and (2) +distributions including binaries display the following +acknowledgement: ``This product includes software developed by the +University of California, Berkeley and its contributors'' in the +documentation or other materials provided with the distribution and +in all advertising materials mentioning features or use of this +software. Neither the name of the University nor the names of its +contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. +@end titlepage + +@ifinfo + +@node Top, Name, (dir), (dir) +@top flex + +@cindex scanner generator + +This manual documents @code{flex}. It covers release @value{VERSION}. + +@menu +* Name:: Name +* Synopsis:: Synopsis +* Overview:: Overview +* Description:: Description +* Examples:: Some simple examples +* Format:: Format of the input file +* Patterns:: Patterns +* Matching:: How the input is matched +* Actions:: Actions +* Generated scanner:: The generated scanner +* Start conditions:: Start conditions +* Multiple buffers:: Multiple input buffers +* End-of-file rules:: End-of-file rules +* Miscellaneous:: Miscellaneous macros +* User variables:: Values available to the user +* YACC interface:: Interfacing with @code{yacc} +* Options:: Options +* Performance:: Performance considerations +* C++:: Generating C++ scanners +* Incompatibilities:: Incompatibilities with @code{lex} and POSIX +* Diagnostics:: Diagnostics +* Files:: Files +* Deficiencies:: Deficiencies / Bugs +* See also:: See also +* Author:: Author +@c * Index:: Index +@end menu + +@end ifinfo + +@node Name, Synopsis, Top, Top +@section Name + +flex - fast lexical analyzer generator + +@node Synopsis, Overview, Name, Top +@section Synopsis + +@example +flex [-bcdfhilnpstvwBFILTV78+? -C[aefFmr] -ooutput -Pprefix -Sskeleton] +[--help --version] [@var{filename} @dots{}] +@end example + +@node Overview, Description, Synopsis, Top +@section Overview + +This manual describes @code{flex}, a tool for generating programs +that perform pattern-matching on text. The manual +includes both tutorial and reference sections: + +@table @asis +@item Description +a brief overview of the tool + +@item Some Simple Examples + +@item Format Of The Input File + +@item Patterns +the extended regular expressions used by flex + +@item How The Input Is Matched +the rules for determining what has been matched + +@item Actions +how to specify what to do when a pattern is matched + +@item The Generated Scanner +details regarding the scanner that flex produces; +how to control the input source + +@item Start Conditions +introducing context into your scanners, and +managing "mini-scanners" + +@item Multiple Input Buffers +how to manipulate multiple input sources; how to +scan from strings instead of files + +@item End-of-file Rules +special rules for matching the end of the input + +@item Miscellaneous Macros +a summary of macros available to the actions + +@item Values Available To The User +a summary of values available to the actions + +@item Interfacing With Yacc +connecting flex scanners together with yacc parsers + +@item Options +flex command-line options, and the "%option" +directive + +@item Performance Considerations +how to make your scanner go as fast as possible + +@item Generating C++ Scanners +the (experimental) facility for generating C++ +scanner classes + +@item Incompatibilities With Lex And POSIX +how flex differs from AT&T lex and the POSIX lex +standard + +@item Diagnostics +those error messages produced by flex (or scanners +it generates) whose meanings might not be apparent + +@item Files +files used by flex + +@item Deficiencies / Bugs +known problems with flex + +@item See Also +other documentation, related tools + +@item Author +includes contact information +@end table + +@node Description, Examples, Overview, Top +@section Description + +@code{flex} is a tool for generating @dfn{scanners}: programs which +recognized lexical patterns in text. @code{flex} reads the given +input files, or its standard input if no file names are +given, for a description of a scanner to generate. The +description is in the form of pairs of regular expressions +and C code, called @dfn{rules}. @code{flex} generates as output a C +source file, @file{lex.yy.c}, which defines a routine @samp{yylex()}. +This file is compiled and linked with the @samp{-lfl} library to +produce an executable. When the executable is run, it +analyzes its input for occurrences of the regular +expressions. Whenever it finds one, it executes the +corresponding C code. + +@node Examples, Format, Description, Top +@section Some simple examples + +First some simple examples to get the flavor of how one +uses @code{flex}. The following @code{flex} input specifies a scanner +which whenever it encounters the string "username" will +replace it with the user's login name: + +@example +%% +username printf( "%s", getlogin() ); +@end example + +By default, any text not matched by a @code{flex} scanner is +copied to the output, so the net effect of this scanner is +to copy its input file to its output with each occurrence +of "username" expanded. In this input, there is just one +rule. "username" is the @var{pattern} and the "printf" is the +@var{action}. The "%%" marks the beginning of the rules. + +Here's another simple example: + +@example + int num_lines = 0, num_chars = 0; + +%% +\n ++num_lines; ++num_chars; +. ++num_chars; + +%% +main() + @{ + yylex(); + printf( "# of lines = %d, # of chars = %d\n", + num_lines, num_chars ); + @} +@end example + +This scanner counts the number of characters and the +number of lines in its input (it produces no output other +than the final report on the counts). The first line +declares two globals, "num_lines" and "num_chars", which +are accessible both inside @samp{yylex()} and in the @samp{main()} +routine declared after the second "%%". There are two rules, +one which matches a newline ("\n") and increments both the +line count and the character count, and one which matches +any character other than a newline (indicated by the "." +regular expression). + +A somewhat more complicated example: + +@example +/* scanner for a toy Pascal-like language */ + +%@{ +/* need this for the call to atof() below */ +#include <math.h> +%@} + +DIGIT [0-9] +ID [a-z][a-z0-9]* + +%% + +@{DIGIT@}+ @{ + printf( "An integer: %s (%d)\n", yytext, + atoi( yytext ) ); + @} + +@{DIGIT@}+"."@{DIGIT@}* @{ + printf( "A float: %s (%g)\n", yytext, + atof( yytext ) ); + @} + +if|then|begin|end|procedure|function @{ + printf( "A keyword: %s\n", yytext ); + @} + +@{ID@} printf( "An identifier: %s\n", yytext ); + +"+"|"-"|"*"|"/" printf( "An operator: %s\n", yytext ); + +"@{"[^@}\n]*"@}" /* eat up one-line comments */ + +[ \t\n]+ /* eat up whitespace */ + +. printf( "Unrecognized character: %s\n", yytext ); + +%% + +main( argc, argv ) +int argc; +char **argv; + @{ + ++argv, --argc; /* skip over program name */ + if ( argc > 0 ) + yyin = fopen( argv[0], "r" ); + else + yyin = stdin; + + yylex(); + @} +@end example + +This is the beginnings of a simple scanner for a language +like Pascal. It identifies different types of @var{tokens} and +reports on what it has seen. + +The details of this example will be explained in the +following sections. + +@node Format, Patterns, Examples, Top +@section Format of the input file + +The @code{flex} input file consists of three sections, separated +by a line with just @samp{%%} in it: + +@example +definitions +%% +rules +%% +user code +@end example + +The @dfn{definitions} section contains declarations of simple +@dfn{name} definitions to simplify the scanner specification, +and declarations of @dfn{start conditions}, which are explained +in a later section. +Name definitions have the form: + +@example +name definition +@end example + +The "name" is a word beginning with a letter or an +underscore ('_') followed by zero or more letters, digits, '_', +or '-' (dash). The definition is taken to begin at the +first non-white-space character following the name and +continuing to the end of the line. The definition can +subsequently be referred to using "@{name@}", which will +expand to "(definition)". For example, + +@example +DIGIT [0-9] +ID [a-z][a-z0-9]* +@end example + +@noindent +defines "DIGIT" to be a regular expression which matches a +single digit, and "ID" to be a regular expression which +matches a letter followed by zero-or-more +letters-or-digits. A subsequent reference to + +@example +@{DIGIT@}+"."@{DIGIT@}* +@end example + +@noindent +is identical to + +@example +([0-9])+"."([0-9])* +@end example + +@noindent +and matches one-or-more digits followed by a '.' followed +by zero-or-more digits. + +The @var{rules} section of the @code{flex} input contains a series of +rules of the form: + +@example +pattern action +@end example + +@noindent +where the pattern must be unindented and the action must +begin on the same line. + +See below for a further description of patterns and +actions. + +Finally, the user code section is simply copied to +@file{lex.yy.c} verbatim. It is used for companion routines +which call or are called by the scanner. The presence of +this section is optional; if it is missing, the second @samp{%%} +in the input file may be skipped, too. + +In the definitions and rules sections, any @emph{indented} text or +text enclosed in @samp{%@{} and @samp{%@}} is copied verbatim to the +output (with the @samp{%@{@}}'s removed). The @samp{%@{@}}'s must +appear unindented on lines by themselves. + +In the rules section, any indented or %@{@} text appearing +before the first rule may be used to declare variables +which are local to the scanning routine and (after the +declarations) code which is to be executed whenever the +scanning routine is entered. Other indented or %@{@} text +in the rule section is still copied to the output, but its +meaning is not well-defined and it may well cause +compile-time errors (this feature is present for @code{POSIX} compliance; +see below for other such features). + +In the definitions section (but not in the rules section), +an unindented comment (i.e., a line beginning with "/*") +is also copied verbatim to the output up to the next "*/". + +@node Patterns, Matching, Format, Top +@section Patterns + +The patterns in the input are written using an extended +set of regular expressions. These are: + +@table @samp +@item x +match the character @samp{x} +@item . +any character (byte) except newline +@item [xyz] +a "character class"; in this case, the pattern +matches either an @samp{x}, a @samp{y}, or a @samp{z} +@item [abj-oZ] +a "character class" with a range in it; matches +an @samp{a}, a @samp{b}, any letter from @samp{j} through @samp{o}, +or a @samp{Z} +@item [^A-Z] +a "negated character class", i.e., any character +but those in the class. In this case, any +character EXCEPT an uppercase letter. +@item [^A-Z\n] +any character EXCEPT an uppercase letter or +a newline +@item @var{r}* +zero or more @var{r}'s, where @var{r} is any regular expression +@item @var{r}+ +one or more @var{r}'s +@item @var{r}? +zero or one @var{r}'s (that is, "an optional @var{r}") +@item @var{r}@{2,5@} +anywhere from two to five @var{r}'s +@item @var{r}@{2,@} +two or more @var{r}'s +@item @var{r}@{4@} +exactly 4 @var{r}'s +@item @{@var{name}@} +the expansion of the "@var{name}" definition +(see above) +@item "[xyz]\"foo" +the literal string: @samp{[xyz]"foo} +@item \@var{x} +if @var{x} is an @samp{a}, @samp{b}, @samp{f}, @samp{n}, @samp{r}, @samp{t}, or @samp{v}, +then the ANSI-C interpretation of \@var{x}. +Otherwise, a literal @samp{@var{x}} (used to escape +operators such as @samp{*}) +@item \0 +a NUL character (ASCII code 0) +@item \123 +the character with octal value 123 +@item \x2a +the character with hexadecimal value @code{2a} +@item (@var{r}) +match an @var{r}; parentheses are used to override +precedence (see below) +@item @var{r}@var{s} +the regular expression @var{r} followed by the +regular expression @var{s}; called "concatenation" +@item @var{r}|@var{s} +either an @var{r} or an @var{s} +@item @var{r}/@var{s} +an @var{r} but only if it is followed by an @var{s}. The text +matched by @var{s} is included when determining whether this rule is +the @dfn{longest match}, but is then returned to the input before +the action is executed. So the action only sees the text matched +by @var{r}. This type of pattern is called @dfn{trailing context}. +(There are some combinations of @samp{@var{r}/@var{s}} that @code{flex} +cannot match correctly; see notes in the Deficiencies / Bugs section +below regarding "dangerous trailing context".) +@item ^@var{r} +an @var{r}, but only at the beginning of a line (i.e., +which just starting to scan, or right after a +newline has been scanned). +@item @var{r}$ +an @var{r}, but only at the end of a line (i.e., just +before a newline). Equivalent to "@var{r}/\n". + +Note that flex's notion of "newline" is exactly +whatever the C compiler used to compile flex +interprets '\n' as; in particular, on some DOS +systems you must either filter out \r's in the +input yourself, or explicitly use @var{r}/\r\n for "r$". +@item <@var{s}>@var{r} +an @var{r}, but only in start condition @var{s} (see +below for discussion of start conditions) +<@var{s1},@var{s2},@var{s3}>@var{r} +same, but in any of start conditions @var{s1}, +@var{s2}, or @var{s3} +@item <*>@var{r} +an @var{r} in any start condition, even an exclusive one. +@item <<EOF>> +an end-of-file +<@var{s1},@var{s2}><<EOF>> +an end-of-file when in start condition @var{s1} or @var{s2} +@end table + +Note that inside of a character class, all regular +expression operators lose their special meaning except escape +('\') and the character class operators, '-', ']', and, at +the beginning of the class, '^'. + +The regular expressions listed above are grouped according +to precedence, from highest precedence at the top to +lowest at the bottom. Those grouped together have equal +precedence. For example, + +@example +foo|bar* +@end example + +@noindent +is the same as + +@example +(foo)|(ba(r*)) +@end example + +@noindent +since the '*' operator has higher precedence than +concatenation, and concatenation higher than alternation ('|'). +This pattern therefore matches @emph{either} the string "foo" @emph{or} +the string "ba" followed by zero-or-more r's. To match +"foo" or zero-or-more "bar"'s, use: + +@example +foo|(bar)* +@end example + +@noindent +and to match zero-or-more "foo"'s-or-"bar"'s: + +@example +(foo|bar)* +@end example + +In addition to characters and ranges of characters, +character classes can also contain character class +@dfn{expressions}. These are expressions enclosed inside @samp{[}: and @samp{:}] +delimiters (which themselves must appear between the '[' +and ']' of the character class; other elements may occur +inside the character class, too). The valid expressions +are: + +@example +[:alnum:] [:alpha:] [:blank:] +[:cntrl:] [:digit:] [:graph:] +[:lower:] [:print:] [:punct:] +[:space:] [:upper:] [:xdigit:] +@end example + +These expressions all designate a set of characters +equivalent to the corresponding standard C @samp{isXXX} function. For +example, @samp{[:alnum:]} designates those characters for which +@samp{isalnum()} returns true - i.e., any alphabetic or numeric. +Some systems don't provide @samp{isblank()}, so flex defines +@samp{[:blank:]} as a blank or a tab. + +For example, the following character classes are all +equivalent: + +@example +[[:alnum:]] +[[:alpha:][:digit:] +[[:alpha:]0-9] +[a-zA-Z0-9] +@end example + +If your scanner is case-insensitive (the @samp{-i} flag), then +@samp{[:upper:]} and @samp{[:lower:]} are equivalent to @samp{[:alpha:]}. + +Some notes on patterns: + +@itemize - +@item +A negated character class such as the example +"[^A-Z]" above @emph{will match a newline} unless "\n" (or an +equivalent escape sequence) is one of the +characters explicitly present in the negated character +class (e.g., "[^A-Z\n]"). This is unlike how many +other regular expression tools treat negated +character classes, but unfortunately the inconsistency +is historically entrenched. Matching newlines +means that a pattern like [^"]* can match the +entire input unless there's another quote in the +input. + +@item +A rule can have at most one instance of trailing +context (the '/' operator or the '$' operator). +The start condition, '^', and "<<EOF>>" patterns +can only occur at the beginning of a pattern, and, +as well as with '/' and '$', cannot be grouped +inside parentheses. A '^' which does not occur at +the beginning of a rule or a '$' which does not +occur at the end of a rule loses its special +properties and is treated as a normal character. + +The following are illegal: + +@example +foo/bar$ +<sc1>foo<sc2>bar +@end example + +Note that the first of these, can be written +"foo/bar\n". + +The following will result in '$' or '^' being +treated as a normal character: + +@example +foo|(bar$) +foo|^bar +@end example + +If what's wanted is a "foo" or a +bar-followed-by-a-newline, the following could be used (the special +'|' action is explained below): + +@example +foo | +bar$ /* action goes here */ +@end example + +A similar trick will work for matching a foo or a +bar-at-the-beginning-of-a-line. +@end itemize + +@node Matching, Actions, Patterns, Top +@section How the input is matched + +When the generated scanner is run, it analyzes its input +looking for strings which match any of its patterns. If +it finds more than one match, it takes the one matching +the most text (for trailing context rules, this includes +the length of the trailing part, even though it will then +be returned to the input). If it finds two or more +matches of the same length, the rule listed first in the +@code{flex} input file is chosen. + +Once the match is determined, the text corresponding to +the match (called the @var{token}) is made available in the +global character pointer @code{yytext}, and its length in the +global integer @code{yyleng}. The @var{action} corresponding to the +matched pattern is then executed (a more detailed +description of actions follows), and then the remaining input is +scanned for another match. + +If no match is found, then the @dfn{default rule} is executed: +the next character in the input is considered matched and +copied to the standard output. Thus, the simplest legal +@code{flex} input is: + +@example +%% +@end example + +which generates a scanner that simply copies its input +(one character at a time) to its output. + +Note that @code{yytext} can be defined in two different ways: +either as a character @emph{pointer} or as a character @emph{array}. +You can control which definition @code{flex} uses by including +one of the special directives @samp{%pointer} or @samp{%array} in the +first (definitions) section of your flex input. The +default is @samp{%pointer}, unless you use the @samp{-l} lex +compatibility option, in which case @code{yytext} will be an array. The +advantage of using @samp{%pointer} is substantially faster +scanning and no buffer overflow when matching very large +tokens (unless you run out of dynamic memory). The +disadvantage is that you are restricted in how your actions can +modify @code{yytext} (see the next section), and calls to the +@samp{unput()} function destroys the present contents of @code{yytext}, +which can be a considerable porting headache when moving +between different @code{lex} versions. + +The advantage of @samp{%array} is that you can then modify @code{yytext} +to your heart's content, and calls to @samp{unput()} do not +destroy @code{yytext} (see below). Furthermore, existing @code{lex} +programs sometimes access @code{yytext} externally using +declarations of the form: +@example +extern char yytext[]; +@end example +This definition is erroneous when used with @samp{%pointer}, but +correct for @samp{%array}. + +@samp{%array} defines @code{yytext} to be an array of @code{YYLMAX} characters, +which defaults to a fairly large value. You can change +the size by simply #define'ing @code{YYLMAX} to a different value +in the first section of your @code{flex} input. As mentioned +above, with @samp{%pointer} yytext grows dynamically to +accommodate large tokens. While this means your @samp{%pointer} scanner +can accommodate very large tokens (such as matching entire +blocks of comments), bear in mind that each time the +scanner must resize @code{yytext} it also must rescan the entire +token from the beginning, so matching such tokens can +prove slow. @code{yytext} presently does @emph{not} dynamically grow if +a call to @samp{unput()} results in too much text being pushed +back; instead, a run-time error results. + +Also note that you cannot use @samp{%array} with C++ scanner +classes (the @code{c++} option; see below). + +@node Actions, Generated scanner, Matching, Top +@section Actions + +Each pattern in a rule has a corresponding action, which +can be any arbitrary C statement. The pattern ends at the +first non-escaped whitespace character; the remainder of +the line is its action. If the action is empty, then when +the pattern is matched the input token is simply +discarded. For example, here is the specification for a +program which deletes all occurrences of "zap me" from its +input: + +@example +%% +"zap me" +@end example + +(It will copy all other characters in the input to the +output since they will be matched by the default rule.) + +Here is a program which compresses multiple blanks and +tabs down to a single blank, and throws away whitespace +found at the end of a line: + +@example +%% +[ \t]+ putchar( ' ' ); +[ \t]+$ /* ignore this token */ +@end example + +If the action contains a '@{', then the action spans till +the balancing '@}' is found, and the action may cross +multiple lines. @code{flex} knows about C strings and comments and +won't be fooled by braces found within them, but also +allows actions to begin with @samp{%@{} and will consider the +action to be all the text up to the next @samp{%@}} (regardless of +ordinary braces inside the action). + +An action consisting solely of a vertical bar ('|') means +"same as the action for the next rule." See below for an +illustration. + +Actions can include arbitrary C code, including @code{return} +statements to return a value to whatever routine called +@samp{yylex()}. Each time @samp{yylex()} is called it continues +processing tokens from where it last left off until it either +reaches the end of the file or executes a return. + +Actions are free to modify @code{yytext} except for lengthening +it (adding characters to its end--these will overwrite +later characters in the input stream). This however does +not apply when using @samp{%array} (see above); in that case, +@code{yytext} may be freely modified in any way. + +Actions are free to modify @code{yyleng} except they should not +do so if the action also includes use of @samp{yymore()} (see +below). + +There are a number of special directives which can be +included within an action: + +@itemize - +@item +@samp{ECHO} copies yytext to the scanner's output. + +@item +@code{BEGIN} followed by the name of a start condition +places the scanner in the corresponding start +condition (see below). + +@item +@code{REJECT} directs the scanner to proceed on to the +"second best" rule which matched the input (or a +prefix of the input). The rule is chosen as +described above in "How the Input is Matched", and +@code{yytext} and @code{yyleng} set up appropriately. It may +either be one which matched as much text as the +originally chosen rule but came later in the @code{flex} +input file, or one which matched less text. For +example, the following will both count the words in +the input and call the routine special() whenever +"frob" is seen: + +@example + int word_count = 0; +%% + +frob special(); REJECT; +[^ \t\n]+ ++word_count; +@end example + +Without the @code{REJECT}, any "frob"'s in the input would +not be counted as words, since the scanner normally +executes only one action per token. Multiple +@code{REJECT's} are allowed, each one finding the next +best choice to the currently active rule. For +example, when the following scanner scans the token +"abcd", it will write "abcdabcaba" to the output: + +@example +%% +a | +ab | +abc | +abcd ECHO; REJECT; +.|\n /* eat up any unmatched character */ +@end example + +(The first three rules share the fourth's action +since they use the special '|' action.) @code{REJECT} is +a particularly expensive feature in terms of +scanner performance; if it is used in @emph{any} of the +scanner's actions it will slow down @emph{all} of the +scanner's matching. Furthermore, @code{REJECT} cannot be used +with the @samp{-Cf} or @samp{-CF} options (see below). + +Note also that unlike the other special actions, +@code{REJECT} is a @emph{branch}; code immediately following it +in the action will @emph{not} be executed. + +@item +@samp{yymore()} tells the scanner that the next time it +matches a rule, the corresponding token should be +@emph{appended} onto the current value of @code{yytext} rather +than replacing it. For example, given the input +"mega-kludge" the following will write +"mega-mega-kludge" to the output: + +@example +%% +mega- ECHO; yymore(); +kludge ECHO; +@end example + +First "mega-" is matched and echoed to the output. +Then "kludge" is matched, but the previous "mega-" +is still hanging around at the beginning of @code{yytext} +so the @samp{ECHO} for the "kludge" rule will actually +write "mega-kludge". +@end itemize + +Two notes regarding use of @samp{yymore()}. First, @samp{yymore()} +depends on the value of @code{yyleng} correctly reflecting the +size of the current token, so you must not modify @code{yyleng} +if you are using @samp{yymore()}. Second, the presence of +@samp{yymore()} in the scanner's action entails a minor +performance penalty in the scanner's matching speed. + +@itemize - +@item +@samp{yyless(n)} returns all but the first @var{n} characters of +the current token back to the input stream, where +they will be rescanned when the scanner looks for +the next match. @code{yytext} and @code{yyleng} are adjusted +appropriately (e.g., @code{yyleng} will now be equal to @var{n} +). For example, on the input "foobar" the +following will write out "foobarbar": + +@example +%% +foobar ECHO; yyless(3); +[a-z]+ ECHO; +@end example + +An argument of 0 to @code{yyless} will cause the entire +current input string to be scanned again. Unless +you've changed how the scanner will subsequently +process its input (using @code{BEGIN}, for example), this +will result in an endless loop. + +Note that @code{yyless} is a macro and can only be used in the +flex input file, not from other source files. + +@item +@samp{unput(c)} puts the character @code{c} back onto the input +stream. It will be the next character scanned. +The following action will take the current token +and cause it to be rescanned enclosed in +parentheses. + +@example +@{ +int i; +/* Copy yytext because unput() trashes yytext */ +char *yycopy = strdup( yytext ); +unput( ')' ); +for ( i = yyleng - 1; i >= 0; --i ) + unput( yycopy[i] ); +unput( '(' ); +free( yycopy ); +@} +@end example + +Note that since each @samp{unput()} puts the given +character back at the @emph{beginning} of the input stream, +pushing back strings must be done back-to-front. +An important potential problem when using @samp{unput()} is that +if you are using @samp{%pointer} (the default), a call to @samp{unput()} +@emph{destroys} the contents of @code{yytext}, starting with its +rightmost character and devouring one character to the left +with each call. If you need the value of yytext preserved +after a call to @samp{unput()} (as in the above example), you +must either first copy it elsewhere, or build your scanner +using @samp{%array} instead (see How The Input Is Matched). + +Finally, note that you cannot put back @code{EOF} to attempt to +mark the input stream with an end-of-file. + +@item +@samp{input()} reads the next character from the input +stream. For example, the following is one way to +eat up C comments: + +@example +%% +"/*" @{ + register int c; + + for ( ; ; ) + @{ + while ( (c = input()) != '*' && + c != EOF ) + ; /* eat up text of comment */ + + if ( c == '*' ) + @{ + while ( (c = input()) == '*' ) + ; + if ( c == '/' ) + break; /* found the end */ + @} + + if ( c == EOF ) + @{ + error( "EOF in comment" ); + break; + @} + @} + @} +@end example + +(Note that if the scanner is compiled using @samp{C++}, +then @samp{input()} is instead referred to as @samp{yyinput()}, +in order to avoid a name clash with the @samp{C++} stream +by the name of @code{input}.) + +@item YY_FLUSH_BUFFER +flushes the scanner's internal buffer so that the next time the scanner +attempts to match a token, it will first refill the buffer using +@code{YY_INPUT} (see The Generated Scanner, below). This action is +a special case of the more general @samp{yy_flush_buffer()} function, +described below in the section Multiple Input Buffers. + +@item +@samp{yyterminate()} can be used in lieu of a return +statement in an action. It terminates the scanner +and returns a 0 to the scanner's caller, indicating +"all done". By default, @samp{yyterminate()} is also +called when an end-of-file is encountered. It is a +macro and may be redefined. +@end itemize + +@node Generated scanner, Start conditions, Actions, Top +@section The generated scanner + +The output of @code{flex} is the file @file{lex.yy.c}, which contains +the scanning routine @samp{yylex()}, a number of tables used by +it for matching tokens, and a number of auxiliary routines +and macros. By default, @samp{yylex()} is declared as follows: + +@example +int yylex() + @{ + @dots{} various definitions and the actions in here @dots{} + @} +@end example + +(If your environment supports function prototypes, then it +will be "int yylex( void )".) This definition may be +changed by defining the "YY_DECL" macro. For example, you +could use: + +@example +#define YY_DECL float lexscan( a, b ) float a, b; +@end example + +to give the scanning routine the name @code{lexscan}, returning a +float, and taking two floats as arguments. Note that if +you give arguments to the scanning routine using a +K&R-style/non-prototyped function declaration, you must +terminate the definition with a semi-colon (@samp{;}). + +Whenever @samp{yylex()} is called, it scans tokens from the +global input file @code{yyin} (which defaults to stdin). It +continues until it either reaches an end-of-file (at which +point it returns the value 0) or one of its actions +executes a @code{return} statement. + +If the scanner reaches an end-of-file, subsequent calls are undefined +unless either @code{yyin} is pointed at a new input file (in which case +scanning continues from that file), or @samp{yyrestart()} is called. +@samp{yyrestart()} takes one argument, a @samp{FILE *} pointer (which +can be nil, if you've set up @code{YY_INPUT} to scan from a source +other than @code{yyin}), and initializes @code{yyin} for scanning from +that file. Essentially there is no difference between just assigning +@code{yyin} to a new input file or using @samp{yyrestart()} to do so; +the latter is available for compatibility with previous versions of +@code{flex}, and because it can be used to switch input files in the +middle of scanning. It can also be used to throw away the current +input buffer, by calling it with an argument of @code{yyin}; but +better is to use @code{YY_FLUSH_BUFFER} (see above). Note that +@samp{yyrestart()} does @emph{not} reset the start condition to +@code{INITIAL} (see Start Conditions, below). + + +If @samp{yylex()} stops scanning due to executing a @code{return} +statement in one of the actions, the scanner may then be called +again and it will resume scanning where it left off. + +By default (and for purposes of efficiency), the scanner +uses block-reads rather than simple @samp{getc()} calls to read +characters from @code{yyin}. The nature of how it gets its input +can be controlled by defining the @code{YY_INPUT} macro. +YY_INPUT's calling sequence is +"YY_INPUT(buf,result,max_size)". Its action is to place +up to @var{max_size} characters in the character array @var{buf} and +return in the integer variable @var{result} either the number of +characters read or the constant YY_NULL (0 on Unix +systems) to indicate EOF. The default YY_INPUT reads from +the global file-pointer "yyin". + +A sample definition of YY_INPUT (in the definitions +section of the input file): + +@example +%@{ +#define YY_INPUT(buf,result,max_size) \ + @{ \ + int c = getchar(); \ + result = (c == EOF) ? YY_NULL : (buf[0] = c, 1); \ + @} +%@} +@end example + +This definition will change the input processing to occur +one character at a time. + +When the scanner receives an end-of-file indication from +YY_INPUT, it then checks the @samp{yywrap()} function. If +@samp{yywrap()} returns false (zero), then it is assumed that the +function has gone ahead and set up @code{yyin} to point to +another input file, and scanning continues. If it returns +true (non-zero), then the scanner terminates, returning 0 +to its caller. Note that in either case, the start +condition remains unchanged; it does @emph{not} revert to @code{INITIAL}. + +If you do not supply your own version of @samp{yywrap()}, then you +must either use @samp{%option noyywrap} (in which case the scanner +behaves as though @samp{yywrap()} returned 1), or you must link with +@samp{-lfl} to obtain the default version of the routine, which always +returns 1. + +Three routines are available for scanning from in-memory +buffers rather than files: @samp{yy_scan_string()}, +@samp{yy_scan_bytes()}, and @samp{yy_scan_buffer()}. See the discussion +of them below in the section Multiple Input Buffers. + +The scanner writes its @samp{ECHO} output to the @code{yyout} global +(default, stdout), which may be redefined by the user +simply by assigning it to some other @code{FILE} pointer. + +@node Start conditions, Multiple buffers, Generated scanner, Top +@section Start conditions + +@code{flex} provides a mechanism for conditionally activating +rules. Any rule whose pattern is prefixed with "<sc>" +will only be active when the scanner is in the start +condition named "sc". For example, + +@example +<STRING>[^"]* @{ /* eat up the string body ... */ + @dots{} + @} +@end example + +@noindent +will be active only when the scanner is in the "STRING" +start condition, and + +@example +<INITIAL,STRING,QUOTE>\. @{ /* handle an escape ... */ + @dots{} + @} +@end example + +@noindent +will be active only when the current start condition is +either "INITIAL", "STRING", or "QUOTE". + +Start conditions are declared in the definitions (first) +section of the input using unindented lines beginning with +either @samp{%s} or @samp{%x} followed by a list of names. The former +declares @emph{inclusive} start conditions, the latter @emph{exclusive} +start conditions. A start condition is activated using +the @code{BEGIN} action. Until the next @code{BEGIN} action is +executed, rules with the given start condition will be active +and rules with other start conditions will be inactive. +If the start condition is @emph{inclusive}, then rules with no +start conditions at all will also be active. If it is +@emph{exclusive}, then @emph{only} rules qualified with the start +condition will be active. A set of rules contingent on the +same exclusive start condition describe a scanner which is +independent of any of the other rules in the @code{flex} input. +Because of this, exclusive start conditions make it easy +to specify "mini-scanners" which scan portions of the +input that are syntactically different from the rest +(e.g., comments). + +If the distinction between inclusive and exclusive start +conditions is still a little vague, here's a simple +example illustrating the connection between the two. The set +of rules: + +@example +%s example +%% + +<example>foo do_something(); + +bar something_else(); +@end example + +@noindent +is equivalent to + +@example +%x example +%% + +<example>foo do_something(); + +<INITIAL,example>bar something_else(); +@end example + +Without the @samp{<INITIAL,example>} qualifier, the @samp{bar} pattern +in the second example wouldn't be active (i.e., couldn't match) when +in start condition @samp{example}. If we just used @samp{<example>} +to qualify @samp{bar}, though, then it would only be active in +@samp{example} and not in @code{INITIAL}, while in the first example +it's active in both, because in the first example the @samp{example} +starting condition is an @emph{inclusive} (@samp{%s}) start condition. + +Also note that the special start-condition specifier @samp{<*>} +matches every start condition. Thus, the above example +could also have been written; + +@example +%x example +%% + +<example>foo do_something(); + +<*>bar something_else(); +@end example + +The default rule (to @samp{ECHO} any unmatched character) remains +active in start conditions. It is equivalent to: + +@example +<*>.|\\n ECHO; +@end example + +@samp{BEGIN(0)} returns to the original state where only the +rules with no start conditions are active. This state can +also be referred to as the start-condition "INITIAL", so +@samp{BEGIN(INITIAL)} is equivalent to @samp{BEGIN(0)}. (The +parentheses around the start condition name are not required but +are considered good style.) + +@code{BEGIN} actions can also be given as indented code at the +beginning of the rules section. For example, the +following will cause the scanner to enter the "SPECIAL" start +condition whenever @samp{yylex()} is called and the global +variable @code{enter_special} is true: + +@example + int enter_special; + +%x SPECIAL +%% + if ( enter_special ) + BEGIN(SPECIAL); + +<SPECIAL>blahblahblah +@dots{}more rules follow@dots{} +@end example + +To illustrate the uses of start conditions, here is a +scanner which provides two different interpretations of a +string like "123.456". By default it will treat it as as +three tokens, the integer "123", a dot ('.'), and the +integer "456". But if the string is preceded earlier in +the line by the string "expect-floats" it will treat it as +a single token, the floating-point number 123.456: + +@example +%@{ +#include <math.h> +%@} +%s expect + +%% +expect-floats BEGIN(expect); + +<expect>[0-9]+"."[0-9]+ @{ + printf( "found a float, = %f\n", + atof( yytext ) ); + @} +<expect>\n @{ + /* that's the end of the line, so + * we need another "expect-number" + * before we'll recognize any more + * numbers + */ + BEGIN(INITIAL); + @} + +[0-9]+ @{ + +Version 2.5 December 1994 18 + + printf( "found an integer, = %d\n", + atoi( yytext ) ); + @} + +"." printf( "found a dot\n" ); +@end example + +Here is a scanner which recognizes (and discards) C +comments while maintaining a count of the current input line. + +@example +%x comment +%% + int line_num = 1; + +"/*" BEGIN(comment); + +<comment>[^*\n]* /* eat anything that's not a '*' */ +<comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ +<comment>\n ++line_num; +<comment>"*"+"/" BEGIN(INITIAL); +@end example + +This scanner goes to a bit of trouble to match as much +text as possible with each rule. In general, when +attempting to write a high-speed scanner try to match as +much possible in each rule, as it's a big win. + +Note that start-conditions names are really integer values +and can be stored as such. Thus, the above could be +extended in the following fashion: + +@example +%x comment foo +%% + int line_num = 1; + int comment_caller; + +"/*" @{ + comment_caller = INITIAL; + BEGIN(comment); + @} + +@dots{} + +<foo>"/*" @{ + comment_caller = foo; + BEGIN(comment); + @} + +<comment>[^*\n]* /* eat anything that's not a '*' */ +<comment>"*"+[^*/\n]* /* eat up '*'s not followed by '/'s */ +<comment>\n ++line_num; +<comment>"*"+"/" BEGIN(comment_caller); +@end example + +Furthermore, you can access the current start condition +using the integer-valued @code{YY_START} macro. For example, the +above assignments to @code{comment_caller} could instead be +written + +@example +comment_caller = YY_START; +@end example + +Flex provides @code{YYSTATE} as an alias for @code{YY_START} (since that +is what's used by AT&T @code{lex}). + +Note that start conditions do not have their own +name-space; %s's and %x's declare names in the same fashion as +#define's. + +Finally, here's an example of how to match C-style quoted +strings using exclusive start conditions, including +expanded escape sequences (but not including checking for +a string that's too long): + +@example +%x str + +%% + char string_buf[MAX_STR_CONST]; + char *string_buf_ptr; + +\" string_buf_ptr = string_buf; BEGIN(str); + +<str>\" @{ /* saw closing quote - all done */ + BEGIN(INITIAL); + *string_buf_ptr = '\0'; + /* return string constant token type and + * value to parser + */ + @} + +<str>\n @{ + /* error - unterminated string constant */ + /* generate error message */ + @} + +<str>\\[0-7]@{1,3@} @{ + /* octal escape sequence */ + int result; + + (void) sscanf( yytext + 1, "%o", &result ); + + if ( result > 0xff ) + /* error, constant is out-of-bounds */ + + *string_buf_ptr++ = result; + @} + +<str>\\[0-9]+ @{ + /* generate error - bad escape sequence; something + * like '\48' or '\0777777' + */ + @} + +<str>\\n *string_buf_ptr++ = '\n'; +<str>\\t *string_buf_ptr++ = '\t'; +<str>\\r *string_buf_ptr++ = '\r'; +<str>\\b *string_buf_ptr++ = '\b'; +<str>\\f *string_buf_ptr++ = '\f'; + +<str>\\(.|\n) *string_buf_ptr++ = yytext[1]; + +<str>[^\\\n\"]+ @{ + char *yptr = yytext; + + while ( *yptr ) + *string_buf_ptr++ = *yptr++; + @} +@end example + +Often, such as in some of the examples above, you wind up +writing a whole bunch of rules all preceded by the same +start condition(s). Flex makes this a little easier and +cleaner by introducing a notion of start condition @dfn{scope}. +A start condition scope is begun with: + +@example +<SCs>@{ +@end example + +@noindent +where SCs is a list of one or more start conditions. +Inside the start condition scope, every rule automatically +has the prefix @samp{<SCs>} applied to it, until a @samp{@}} which +matches the initial @samp{@{}. So, for example, + +@example +<ESC>@{ + "\\n" return '\n'; + "\\r" return '\r'; + "\\f" return '\f'; + "\\0" return '\0'; +@} +@end example + +@noindent +is equivalent to: + +@example +<ESC>"\\n" return '\n'; +<ESC>"\\r" return '\r'; +<ESC>"\\f" return '\f'; +<ESC>"\\0" return '\0'; +@end example + +Start condition scopes may be nested. + +Three routines are available for manipulating stacks of +start conditions: + +@table @samp +@item void yy_push_state(int new_state) +pushes the current start condition onto the top of +the start condition stack and switches to @var{new_state} +as though you had used @samp{BEGIN new_state} (recall that +start condition names are also integers). + +@item void yy_pop_state() +pops the top of the stack and switches to it via +@code{BEGIN}. + +@item int yy_top_state() +returns the top of the stack without altering the +stack's contents. +@end table + +The start condition stack grows dynamically and so has no +built-in size limitation. If memory is exhausted, program +execution aborts. + +To use start condition stacks, your scanner must include a +@samp{%option stack} directive (see Options below). + +@node Multiple buffers, End-of-file rules, Start conditions, Top +@section Multiple input buffers + +Some scanners (such as those which support "include" +files) require reading from several input streams. As +@code{flex} scanners do a large amount of buffering, one cannot +control where the next input will be read from by simply +writing a @code{YY_INPUT} which is sensitive to the scanning +context. @code{YY_INPUT} is only called when the scanner reaches +the end of its buffer, which may be a long time after +scanning a statement such as an "include" which requires +switching the input source. + +To negotiate these sorts of problems, @code{flex} provides a +mechanism for creating and switching between multiple +input buffers. An input buffer is created by using: + +@example +YY_BUFFER_STATE yy_create_buffer( FILE *file, int size ) +@end example + +@noindent +which takes a @code{FILE} pointer and a size and creates a buffer +associated with the given file and large enough to hold +@var{size} characters (when in doubt, use @code{YY_BUF_SIZE} for the +size). It returns a @code{YY_BUFFER_STATE} handle, which may +then be passed to other routines (see below). The +@code{YY_BUFFER_STATE} type is a pointer to an opaque @code{struct} +@code{yy_buffer_state} structure, so you may safely initialize +YY_BUFFER_STATE variables to @samp{((YY_BUFFER_STATE) 0)} if you +wish, and also refer to the opaque structure in order to +correctly declare input buffers in source files other than +that of your scanner. Note that the @code{FILE} pointer in the +call to @code{yy_create_buffer} is only used as the value of @code{yyin} +seen by @code{YY_INPUT}; if you redefine @code{YY_INPUT} so it no longer +uses @code{yyin}, then you can safely pass a nil @code{FILE} pointer to +@code{yy_create_buffer}. You select a particular buffer to scan +from using: + +@example +void yy_switch_to_buffer( YY_BUFFER_STATE new_buffer ) +@end example + +switches the scanner's input buffer so subsequent tokens +will come from @var{new_buffer}. Note that +@samp{yy_switch_to_buffer()} may be used by @samp{yywrap()} to set +things up for continued scanning, instead of opening a new +file and pointing @code{yyin} at it. Note also that switching +input sources via either @samp{yy_switch_to_buffer()} or @samp{yywrap()} +does @emph{not} change the start condition. + +@example +void yy_delete_buffer( YY_BUFFER_STATE buffer ) +@end example + +@noindent +is used to reclaim the storage associated with a buffer. +You can also clear the current contents of a buffer using: + +@example +void yy_flush_buffer( YY_BUFFER_STATE buffer ) +@end example + +This function discards the buffer's contents, so the next time the +scanner attempts to match a token from the buffer, it will first fill +the buffer anew using @code{YY_INPUT}. + +@samp{yy_new_buffer()} is an alias for @samp{yy_create_buffer()}, +provided for compatibility with the C++ use of @code{new} and @code{delete} +for creating and destroying dynamic objects. + +Finally, the @code{YY_CURRENT_BUFFER} macro returns a +@code{YY_BUFFER_STATE} handle to the current buffer. + +Here is an example of using these features for writing a +scanner which expands include files (the @samp{<<EOF>>} feature +is discussed below): + +@example +/* the "incl" state is used for picking up the name + * of an include file + */ +%x incl + +%@{ +#define MAX_INCLUDE_DEPTH 10 +YY_BUFFER_STATE include_stack[MAX_INCLUDE_DEPTH]; +int include_stack_ptr = 0; +%@} + +%% +include BEGIN(incl); + +[a-z]+ ECHO; +[^a-z\n]*\n? ECHO; + +<incl>[ \t]* /* eat the whitespace */ +<incl>[^ \t\n]+ @{ /* got the include file name */ + if ( include_stack_ptr >= MAX_INCLUDE_DEPTH ) + @{ + fprintf( stderr, "Includes nested too deeply" ); + exit( 1 ); + @} + + include_stack[include_stack_ptr++] = + YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + + if ( ! yyin ) + error( @dots{} ); + + yy_switch_to_buffer( + yy_create_buffer( yyin, YY_BUF_SIZE ) ); + + BEGIN(INITIAL); + @} + +<<EOF>> @{ + if ( --include_stack_ptr < 0 ) + @{ + yyterminate(); + @} + + else + @{ + yy_delete_buffer( YY_CURRENT_BUFFER ); + yy_switch_to_buffer( + include_stack[include_stack_ptr] ); + @} + @} +@end example + +Three routines are available for setting up input buffers +for scanning in-memory strings instead of files. All of +them create a new input buffer for scanning the string, +and return a corresponding @code{YY_BUFFER_STATE} handle (which +you should delete with @samp{yy_delete_buffer()} when done with +it). They also switch to the new buffer using +@samp{yy_switch_to_buffer()}, so the next call to @samp{yylex()} will +start scanning the string. + +@table @samp +@item yy_scan_string(const char *str) +scans a NUL-terminated string. + +@item yy_scan_bytes(const char *bytes, int len) +scans @code{len} bytes (including possibly NUL's) starting +at location @var{bytes}. +@end table + +Note that both of these functions create and scan a @emph{copy} +of the string or bytes. (This may be desirable, since +@samp{yylex()} modifies the contents of the buffer it is +scanning.) You can avoid the copy by using: + +@table @samp +@item yy_scan_buffer(char *base, yy_size_t size) +which scans in place the buffer starting at @var{base}, +consisting of @var{size} bytes, the last two bytes of +which @emph{must} be @code{YY_END_OF_BUFFER_CHAR} (ASCII NUL). +These last two bytes are not scanned; thus, +scanning consists of @samp{base[0]} through @samp{base[size-2]}, +inclusive. + +If you fail to set up @var{base} in this manner (i.e., +forget the final two @code{YY_END_OF_BUFFER_CHAR} bytes), +then @samp{yy_scan_buffer()} returns a nil pointer instead +of creating a new input buffer. + +The type @code{yy_size_t} is an integral type to which you +can cast an integer expression reflecting the size +of the buffer. +@end table + +@node End-of-file rules, Miscellaneous, Multiple buffers, Top +@section End-of-file rules + +The special rule "<<EOF>>" indicates actions which are to +be taken when an end-of-file is encountered and yywrap() +returns non-zero (i.e., indicates no further files to +process). The action must finish by doing one of four +things: + +@itemize - +@item +assigning @code{yyin} to a new input file (in previous +versions of flex, after doing the assignment you +had to call the special action @code{YY_NEW_FILE}; this is +no longer necessary); + +@item +executing a @code{return} statement; + +@item +executing the special @samp{yyterminate()} action; + +@item +or, switching to a new buffer using +@samp{yy_switch_to_buffer()} as shown in the example +above. +@end itemize + +<<EOF>> rules may not be used with other patterns; they +may only be qualified with a list of start conditions. If +an unqualified <<EOF>> rule is given, it applies to @emph{all} +start conditions which do not already have <<EOF>> +actions. To specify an <<EOF>> rule for only the initial +start condition, use + +@example +<INITIAL><<EOF>> +@end example + +These rules are useful for catching things like unclosed +comments. An example: + +@example +%x quote +%% + +@dots{}other rules for dealing with quotes@dots{} + +<quote><<EOF>> @{ + error( "unterminated quote" ); + yyterminate(); + @} +<<EOF>> @{ + if ( *++filelist ) + yyin = fopen( *filelist, "r" ); + else + yyterminate(); + @} +@end example + +@node Miscellaneous, User variables, End-of-file rules, Top +@section Miscellaneous macros + +The macro @code{YY_USER_ACTION} can be defined to provide an +action which is always executed prior to the matched +rule's action. For example, it could be #define'd to call +a routine to convert yytext to lower-case. When +@code{YY_USER_ACTION} is invoked, the variable @code{yy_act} gives the +number of the matched rule (rules are numbered starting +with 1). Suppose you want to profile how often each of +your rules is matched. The following would do the trick: + +@example +#define YY_USER_ACTION ++ctr[yy_act] +@end example + +where @code{ctr} is an array to hold the counts for the different +rules. Note that the macro @code{YY_NUM_RULES} gives the total number +of rules (including the default rule, even if you use @samp{-s}, so +a correct declaration for @code{ctr} is: + +@example +int ctr[YY_NUM_RULES]; +@end example + +The macro @code{YY_USER_INIT} may be defined to provide an action +which is always executed before the first scan (and before +the scanner's internal initializations are done). For +example, it could be used to call a routine to read in a +data table or open a logging file. + +The macro @samp{yy_set_interactive(is_interactive)} can be used +to control whether the current buffer is considered +@emph{interactive}. An interactive buffer is processed more slowly, +but must be used when the scanner's input source is indeed +interactive to avoid problems due to waiting to fill +buffers (see the discussion of the @samp{-I} flag below). A +non-zero value in the macro invocation marks the buffer as +interactive, a zero value as non-interactive. Note that +use of this macro overrides @samp{%option always-interactive} or +@samp{%option never-interactive} (see Options below). +@samp{yy_set_interactive()} must be invoked prior to beginning to +scan the buffer that is (or is not) to be considered +interactive. + +The macro @samp{yy_set_bol(at_bol)} can be used to control +whether the current buffer's scanning context for the next +token match is done as though at the beginning of a line. +A non-zero macro argument makes rules anchored with + +The macro @samp{YY_AT_BOL()} returns true if the next token +scanned from the current buffer will have '^' rules +active, false otherwise. + +In the generated scanner, the actions are all gathered in +one large switch statement and separated using @code{YY_BREAK}, +which may be redefined. By default, it is simply a +"break", to separate each rule's action from the following +rule's. Redefining @code{YY_BREAK} allows, for example, C++ +users to #define YY_BREAK to do nothing (while being very +careful that every rule ends with a "break" or a +"return"!) to avoid suffering from unreachable statement +warnings where because a rule's action ends with "return", +the @code{YY_BREAK} is inaccessible. + +@node User variables, YACC interface, Miscellaneous, Top +@section Values available to the user + +This section summarizes the various values available to +the user in the rule actions. + +@itemize - +@item +@samp{char *yytext} holds the text of the current token. +It may be modified but not lengthened (you cannot +append characters to the end). + +If the special directive @samp{%array} appears in the +first section of the scanner description, then +@code{yytext} is instead declared @samp{char yytext[YYLMAX]}, +where @code{YYLMAX} is a macro definition that you can +redefine in the first section if you don't like the +default value (generally 8KB). Using @samp{%array} +results in somewhat slower scanners, but the value +of @code{yytext} becomes immune to calls to @samp{input()} and +@samp{unput()}, which potentially destroy its value when +@code{yytext} is a character pointer. The opposite of +@samp{%array} is @samp{%pointer}, which is the default. + +You cannot use @samp{%array} when generating C++ scanner +classes (the @samp{-+} flag). + +@item +@samp{int yyleng} holds the length of the current token. + +@item +@samp{FILE *yyin} is the file which by default @code{flex} reads +from. It may be redefined but doing so only makes +sense before scanning begins or after an EOF has +been encountered. Changing it in the midst of +scanning will have unexpected results since @code{flex} +buffers its input; use @samp{yyrestart()} instead. Once +scanning terminates because an end-of-file has been +seen, you can assign @code{yyin} at the new input file and +then call the scanner again to continue scanning. + +@item +@samp{void yyrestart( FILE *new_file )} may be called to +point @code{yyin} at the new input file. The switch-over +to the new file is immediate (any previously +buffered-up input is lost). Note that calling +@samp{yyrestart()} with @code{yyin} as an argument thus throws +away the current input buffer and continues +scanning the same input file. + +@item +@samp{FILE *yyout} is the file to which @samp{ECHO} actions are +done. It can be reassigned by the user. + +@item +@code{YY_CURRENT_BUFFER} returns a @code{YY_BUFFER_STATE} handle +to the current buffer. + +@item +@code{YY_START} returns an integer value corresponding to +the current start condition. You can subsequently +use this value with @code{BEGIN} to return to that start +condition. +@end itemize + +@node YACC interface, Options, User variables, Top +@section Interfacing with @code{yacc} + +One of the main uses of @code{flex} is as a companion to the @code{yacc} +parser-generator. @code{yacc} parsers expect to call a routine +named @samp{yylex()} to find the next input token. The routine +is supposed to return the type of the next token as well +as putting any associated value in the global @code{yylval}. To +use @code{flex} with @code{yacc}, one specifies the @samp{-d} option to @code{yacc} to +instruct it to generate the file @file{y.tab.h} containing +definitions of all the @samp{%tokens} appearing in the @code{yacc} input. +This file is then included in the @code{flex} scanner. For +example, if one of the tokens is "TOK_NUMBER", part of the +scanner might look like: + +@example +%@{ +#include "y.tab.h" +%@} + +%% + +[0-9]+ yylval = atoi( yytext ); return TOK_NUMBER; +@end example + +@node Options, Performance, YACC interface, Top +@section Options +@code{flex} has the following options: + +@table @samp +@item -b +Generate backing-up information to @file{lex.backup}. +This is a list of scanner states which require +backing up and the input characters on which they +do so. By adding rules one can remove backing-up +states. If @emph{all} backing-up states are eliminated +and @samp{-Cf} or @samp{-CF} is used, the generated scanner will +run faster (see the @samp{-p} flag). Only users who wish +to squeeze every last cycle out of their scanners +need worry about this option. (See the section on +Performance Considerations below.) + +@item -c +is a do-nothing, deprecated option included for +POSIX compliance. + +@item -d +makes the generated scanner run in @dfn{debug} mode. +Whenever a pattern is recognized and the global +@code{yy_flex_debug} is non-zero (which is the default), +the scanner will write to @code{stderr} a line of the +form: + +@example +--accepting rule at line 53 ("the matched text") +@end example + +The line number refers to the location of the rule +in the file defining the scanner (i.e., the file +that was fed to flex). Messages are also generated +when the scanner backs up, accepts the default +rule, reaches the end of its input buffer (or +encounters a NUL; at this point, the two look the +same as far as the scanner's concerned), or reaches +an end-of-file. + +@item -f +specifies @dfn{fast scanner}. No table compression is +done and stdio is bypassed. The result is large +but fast. This option is equivalent to @samp{-Cfr} (see +below). + +@item -h +generates a "help" summary of @code{flex's} options to +@code{stdout} and then exits. @samp{-?} and @samp{--help} are synonyms +for @samp{-h}. + +@item -i +instructs @code{flex} to generate a @emph{case-insensitive} +scanner. The case of letters given in the @code{flex} input +patterns will be ignored, and tokens in the input +will be matched regardless of case. The matched +text given in @code{yytext} will have the preserved case +(i.e., it will not be folded). + +@item -l +turns on maximum compatibility with the original +AT&T @code{lex} implementation. Note that this does not +mean @emph{full} compatibility. Use of this option costs +a considerable amount of performance, and it cannot +be used with the @samp{-+, -f, -F, -Cf}, or @samp{-CF} options. +For details on the compatibilities it provides, see +the section "Incompatibilities With Lex And POSIX" +below. This option also results in the name +@code{YY_FLEX_LEX_COMPAT} being #define'd in the generated +scanner. + +@item -n +is another do-nothing, deprecated option included +only for POSIX compliance. + +@item -p +generates a performance report to stderr. The +report consists of comments regarding features of +the @code{flex} input file which will cause a serious loss +of performance in the resulting scanner. If you +give the flag twice, you will also get comments +regarding features that lead to minor performance +losses. + +Note that the use of @code{REJECT}, @samp{%option yylineno} and +variable trailing context (see the Deficiencies / Bugs section below) +entails a substantial performance penalty; use of @samp{yymore()}, +the @samp{^} operator, and the @samp{-I} flag entail minor performance +penalties. + +@item -s +causes the @dfn{default rule} (that unmatched scanner +input is echoed to @code{stdout}) to be suppressed. If +the scanner encounters input that does not match +any of its rules, it aborts with an error. This +option is useful for finding holes in a scanner's +rule set. + +@item -t +instructs @code{flex} to write the scanner it generates to +standard output instead of @file{lex.yy.c}. + +@item -v +specifies that @code{flex} should write to @code{stderr} a +summary of statistics regarding the scanner it +generates. Most of the statistics are meaningless to +the casual @code{flex} user, but the first line identifies +the version of @code{flex} (same as reported by @samp{-V}), and +the next line the flags used when generating the +scanner, including those that are on by default. + +@item -w +suppresses warning messages. + +@item -B +instructs @code{flex} to generate a @emph{batch} scanner, the +opposite of @emph{interactive} scanners generated by @samp{-I} +(see below). In general, you use @samp{-B} when you are +@emph{certain} that your scanner will never be used +interactively, and you want to squeeze a @emph{little} more +performance out of it. If your goal is instead to +squeeze out a @emph{lot} more performance, you should be +using the @samp{-Cf} or @samp{-CF} options (discussed below), +which turn on @samp{-B} automatically anyway. + +@item -F +specifies that the @dfn{fast} scanner table +representation should be used (and stdio bypassed). This +representation is about as fast as the full table +representation @samp{(-f)}, and for some sets of patterns +will be considerably smaller (and for others, +larger). In general, if the pattern set contains +both "keywords" and a catch-all, "identifier" rule, +such as in the set: + +@example +"case" return TOK_CASE; +"switch" return TOK_SWITCH; +... +"default" return TOK_DEFAULT; +[a-z]+ return TOK_ID; +@end example + +@noindent +then you're better off using the full table +representation. If only the "identifier" rule is +present and you then use a hash table or some such to +detect the keywords, you're better off using @samp{-F}. + +This option is equivalent to @samp{-CFr} (see below). It +cannot be used with @samp{-+}. + +@item -I +instructs @code{flex} to generate an @emph{interactive} scanner. +An interactive scanner is one that only looks ahead +to decide what token has been matched if it +absolutely must. It turns out that always looking one +extra character ahead, even if the scanner has +already seen enough text to disambiguate the +current token, is a bit faster than only looking ahead +when necessary. But scanners that always look +ahead give dreadful interactive performance; for +example, when a user types a newline, it is not +recognized as a newline token until they enter +@emph{another} token, which often means typing in another +whole line. + +@code{Flex} scanners default to @emph{interactive} unless you use +the @samp{-Cf} or @samp{-CF} table-compression options (see +below). That's because if you're looking for +high-performance you should be using one of these +options, so if you didn't, @code{flex} assumes you'd +rather trade off a bit of run-time performance for +intuitive interactive behavior. Note also that you +@emph{cannot} use @samp{-I} in conjunction with @samp{-Cf} or @samp{-CF}. +Thus, this option is not really needed; it is on by +default for all those cases in which it is allowed. + +You can force a scanner to @emph{not} be interactive by +using @samp{-B} (see above). + +@item -L +instructs @code{flex} not to generate @samp{#line} directives. +Without this option, @code{flex} peppers the generated +scanner with #line directives so error messages in +the actions will be correctly located with respect +to either the original @code{flex} input file (if the +errors are due to code in the input file), or +@file{lex.yy.c} (if the errors are @code{flex's} fault -- you +should report these sorts of errors to the email +address given below). + +@item -T +makes @code{flex} run in @code{trace} mode. It will generate a +lot of messages to @code{stderr} concerning the form of +the input and the resultant non-deterministic and +deterministic finite automata. This option is +mostly for use in maintaining @code{flex}. + +@item -V +prints the version number to @code{stdout} and exits. +@samp{--version} is a synonym for @samp{-V}. + +@item -7 +instructs @code{flex} to generate a 7-bit scanner, i.e., +one which can only recognized 7-bit characters in +its input. The advantage of using @samp{-7} is that the +scanner's tables can be up to half the size of +those generated using the @samp{-8} option (see below). +The disadvantage is that such scanners often hang +or crash if their input contains an 8-bit +character. + +Note, however, that unless you generate your +scanner using the @samp{-Cf} or @samp{-CF} table compression options, +use of @samp{-7} will save only a small amount of table +space, and make your scanner considerably less +portable. @code{Flex's} default behavior is to generate +an 8-bit scanner unless you use the @samp{-Cf} or @samp{-CF}, in +which case @code{flex} defaults to generating 7-bit +scanners unless your site was always configured to +generate 8-bit scanners (as will often be the case +with non-USA sites). You can tell whether flex +generated a 7-bit or an 8-bit scanner by inspecting +the flag summary in the @samp{-v} output as described +above. + +Note that if you use @samp{-Cfe} or @samp{-CFe} (those table +compression options, but also using equivalence +classes as discussed see below), flex still +defaults to generating an 8-bit scanner, since +usually with these compression options full 8-bit +tables are not much more expensive than 7-bit +tables. + +@item -8 +instructs @code{flex} to generate an 8-bit scanner, i.e., +one which can recognize 8-bit characters. This +flag is only needed for scanners generated using +@samp{-Cf} or @samp{-CF}, as otherwise flex defaults to +generating an 8-bit scanner anyway. + +See the discussion of @samp{-7} above for flex's default +behavior and the tradeoffs between 7-bit and 8-bit +scanners. + +@item -+ +specifies that you want flex to generate a C++ +scanner class. See the section on Generating C++ +Scanners below for details. + +@item -C[aefFmr] +controls the degree of table compression and, more +generally, trade-offs between small scanners and +fast scanners. + +@samp{-Ca} ("align") instructs flex to trade off larger +tables in the generated scanner for faster +performance because the elements of the tables are better +aligned for memory access and computation. On some +RISC architectures, fetching and manipulating +long-words is more efficient than with smaller-sized +units such as shortwords. This option can double +the size of the tables used by your scanner. + +@samp{-Ce} directs @code{flex} to construct @dfn{equivalence classes}, +i.e., sets of characters which have identical +lexical properties (for example, if the only appearance +of digits in the @code{flex} input is in the character +class "[0-9]" then the digits '0', '1', @dots{}, '9' +will all be put in the same equivalence class). +Equivalence classes usually give dramatic +reductions in the final table/object file sizes +(typically a factor of 2-5) and are pretty cheap +performance-wise (one array look-up per character +scanned). + +@samp{-Cf} specifies that the @emph{full} scanner tables should +be generated - @code{flex} should not compress the tables +by taking advantages of similar transition +functions for different states. + +@samp{-CF} specifies that the alternate fast scanner +representation (described above under the @samp{-F} flag) +should be used. This option cannot be used with +@samp{-+}. + +@samp{-Cm} directs @code{flex} to construct @dfn{meta-equivalence +classes}, which are sets of equivalence classes (or +characters, if equivalence classes are not being +used) that are commonly used together. +Meta-equivalence classes are often a big win when using +compressed tables, but they have a moderate +performance impact (one or two "if" tests and one array +look-up per character scanned). + +@samp{-Cr} causes the generated scanner to @emph{bypass} use of +the standard I/O library (stdio) for input. +Instead of calling @samp{fread()} or @samp{getc()}, the scanner +will use the @samp{read()} system call, resulting in a +performance gain which varies from system to +system, but in general is probably negligible unless +you are also using @samp{-Cf} or @samp{-CF}. Using @samp{-Cr} can cause +strange behavior if, for example, you read from +@code{yyin} using stdio prior to calling the scanner +(because the scanner will miss whatever text your +previous reads left in the stdio input buffer). + +@samp{-Cr} has no effect if you define @code{YY_INPUT} (see The +Generated Scanner above). + +A lone @samp{-C} specifies that the scanner tables should +be compressed but neither equivalence classes nor +meta-equivalence classes should be used. + +The options @samp{-Cf} or @samp{-CF} and @samp{-Cm} do not make sense +together - there is no opportunity for +meta-equivalence classes if the table is not being +compressed. Otherwise the options may be freely +mixed, and are cumulative. + +The default setting is @samp{-Cem}, which specifies that +@code{flex} should generate equivalence classes and +meta-equivalence classes. This setting provides the +highest degree of table compression. You can trade +off faster-executing scanners at the cost of larger +tables with the following generally being true: + +@example +slowest & smallest + -Cem + -Cm + -Ce + -C + -C@{f,F@}e + -C@{f,F@} + -C@{f,F@}a +fastest & largest +@end example + +Note that scanners with the smallest tables are +usually generated and compiled the quickest, so +during development you will usually want to use the +default, maximal compression. + +@samp{-Cfe} is often a good compromise between speed and +size for production scanners. + +@item -ooutput +directs flex to write the scanner to the file @samp{out-} +@code{put} instead of @file{lex.yy.c}. If you combine @samp{-o} with +the @samp{-t} option, then the scanner is written to +@code{stdout} but its @samp{#line} directives (see the @samp{-L} option +above) refer to the file @code{output}. + +@item -Pprefix +changes the default @samp{yy} prefix used by @code{flex} for all +globally-visible variable and function names to +instead be @var{prefix}. For example, @samp{-Pfoo} changes the +name of @code{yytext} to @file{footext}. It also changes the +name of the default output file from @file{lex.yy.c} to +@file{lex.foo.c}. Here are all of the names affected: + +@example +yy_create_buffer +yy_delete_buffer +yy_flex_debug +yy_init_buffer +yy_flush_buffer +yy_load_buffer_state +yy_switch_to_buffer +yyin +yyleng +yylex +yylineno +yyout +yyrestart +yytext +yywrap +@end example + +(If you are using a C++ scanner, then only @code{yywrap} +and @code{yyFlexLexer} are affected.) Within your scanner +itself, you can still refer to the global variables +and functions using either version of their name; +but externally, they have the modified name. + +This option lets you easily link together multiple +@code{flex} programs into the same executable. Note, +though, that using this option also renames +@samp{yywrap()}, so you now @emph{must} either provide your own +(appropriately-named) version of the routine for +your scanner, or use @samp{%option noyywrap}, as linking +with @samp{-lfl} no longer provides one for you by +default. + +@item -Sskeleton_file +overrides the default skeleton file from which @code{flex} +constructs its scanners. You'll never need this +option unless you are doing @code{flex} maintenance or +development. +@end table + +@code{flex} also provides a mechanism for controlling options +within the scanner specification itself, rather than from +the flex command-line. This is done by including @samp{%option} +directives in the first section of the scanner +specification. You can specify multiple options with a single +@samp{%option} directive, and multiple directives in the first +section of your flex input file. Most options are given +simply as names, optionally preceded by the word "no" +(with no intervening whitespace) to negate their meaning. +A number are equivalent to flex flags or their negation: + +@example +7bit -7 option +8bit -8 option +align -Ca option +backup -b option +batch -B option +c++ -+ option + +caseful or +case-sensitive opposite of -i (default) + +case-insensitive or +caseless -i option + +debug -d option +default opposite of -s option +ecs -Ce option +fast -F option +full -f option +interactive -I option +lex-compat -l option +meta-ecs -Cm option +perf-report -p option +read -Cr option +stdout -t option +verbose -v option +warn opposite of -w option + (use "%option nowarn" for -w) + +array equivalent to "%array" +pointer equivalent to "%pointer" (default) +@end example + +Some @samp{%option's} provide features otherwise not available: + +@table @samp +@item always-interactive +instructs flex to generate a scanner which always +considers its input "interactive". Normally, on +each new input file the scanner calls @samp{isatty()} in +an attempt to determine whether the scanner's input +source is interactive and thus should be read a +character at a time. When this option is used, +however, then no such call is made. + +@item main +directs flex to provide a default @samp{main()} program +for the scanner, which simply calls @samp{yylex()}. This +option implies @code{noyywrap} (see below). + +@item never-interactive +instructs flex to generate a scanner which never +considers its input "interactive" (again, no call +made to @samp{isatty())}. This is the opposite of @samp{always-} +@emph{interactive}. + +@item stack +enables the use of start condition stacks (see +Start Conditions above). + +@item stdinit +if unset (i.e., @samp{%option nostdinit}) initializes @code{yyin} +and @code{yyout} to nil @code{FILE} pointers, instead of @code{stdin} +and @code{stdout}. + +@item yylineno +directs @code{flex} to generate a scanner that maintains the number +of the current line read from its input in the global variable +@code{yylineno}. This option is implied by @samp{%option lex-compat}. + +@item yywrap +if unset (i.e., @samp{%option noyywrap}), makes the +scanner not call @samp{yywrap()} upon an end-of-file, but +simply assume that there are no more files to scan +(until the user points @code{yyin} at a new file and calls +@samp{yylex()} again). +@end table + +@code{flex} scans your rule actions to determine whether you use +the @code{REJECT} or @samp{yymore()} features. The @code{reject} and @code{yymore} +options are available to override its decision as to +whether you use the options, either by setting them (e.g., +@samp{%option reject}) to indicate the feature is indeed used, or +unsetting them to indicate it actually is not used (e.g., +@samp{%option noyymore}). + +Three options take string-delimited values, offset with '=': + +@example +%option outfile="ABC" +@end example + +@noindent +is equivalent to @samp{-oABC}, and + +@example +%option prefix="XYZ" +@end example + +@noindent +is equivalent to @samp{-PXYZ}. + +Finally, + +@example +%option yyclass="foo" +@end example + +@noindent +only applies when generating a C++ scanner (@samp{-+} option). It +informs @code{flex} that you have derived @samp{foo} as a subclass of +@code{yyFlexLexer} so @code{flex} will place your actions in the member +function @samp{foo::yylex()} instead of @samp{yyFlexLexer::yylex()}. +It also generates a @samp{yyFlexLexer::yylex()} member function that +emits a run-time error (by invoking @samp{yyFlexLexer::LexerError()}) +if called. See Generating C++ Scanners, below, for additional +information. + +A number of options are available for lint purists who +want to suppress the appearance of unneeded routines in +the generated scanner. Each of the following, if unset, +results in the corresponding routine not appearing in the +generated scanner: + +@example +input, unput +yy_push_state, yy_pop_state, yy_top_state +yy_scan_buffer, yy_scan_bytes, yy_scan_string +@end example + +@noindent +(though @samp{yy_push_state()} and friends won't appear anyway +unless you use @samp{%option stack}). + +@node Performance, C++, Options, Top +@section Performance considerations + +The main design goal of @code{flex} is that it generate +high-performance scanners. It has been optimized for dealing +well with large sets of rules. Aside from the effects on +scanner speed of the table compression @samp{-C} options outlined +above, there are a number of options/actions which degrade +performance. These are, from most expensive to least: + +@example +REJECT +%option yylineno +arbitrary trailing context + +pattern sets that require backing up +%array +%option interactive +%option always-interactive + +'^' beginning-of-line operator +yymore() +@end example + +with the first three all being quite expensive and the +last two being quite cheap. Note also that @samp{unput()} is +implemented as a routine call that potentially does quite +a bit of work, while @samp{yyless()} is a quite-cheap macro; so +if just putting back some excess text you scanned, use +@samp{yyless()}. + +@code{REJECT} should be avoided at all costs when performance is +important. It is a particularly expensive option. + +Getting rid of backing up is messy and often may be an +enormous amount of work for a complicated scanner. In +principal, one begins by using the @samp{-b} flag to generate a +@file{lex.backup} file. For example, on the input + +@example +%% +foo return TOK_KEYWORD; +foobar return TOK_KEYWORD; +@end example + +@noindent +the file looks like: + +@example +State #6 is non-accepting - + associated rule line numbers: + 2 3 + out-transitions: [ o ] + jam-transitions: EOF [ \001-n p-\177 ] + +State #8 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ a ] + jam-transitions: EOF [ \001-` b-\177 ] + +State #9 is non-accepting - + associated rule line numbers: + 3 + out-transitions: [ r ] + jam-transitions: EOF [ \001-q s-\177 ] + +Compressed tables always back up. +@end example + +The first few lines tell us that there's a scanner state +in which it can make a transition on an 'o' but not on any +other character, and that in that state the currently +scanned text does not match any rule. The state occurs +when trying to match the rules found at lines 2 and 3 in +the input file. If the scanner is in that state and then +reads something other than an 'o', it will have to back up +to find a rule which is matched. With a bit of +head-scratching one can see that this must be the state it's in +when it has seen "fo". When this has happened, if +anything other than another 'o' is seen, the scanner will +have to back up to simply match the 'f' (by the default +rule). + +The comment regarding State #8 indicates there's a problem +when "foob" has been scanned. Indeed, on any character +other than an 'a', the scanner will have to back up to +accept "foo". Similarly, the comment for State #9 +concerns when "fooba" has been scanned and an 'r' does not +follow. + +The final comment reminds us that there's no point going +to all the trouble of removing backing up from the rules +unless we're using @samp{-Cf} or @samp{-CF}, since there's no +performance gain doing so with compressed scanners. + +The way to remove the backing up is to add "error" rules: + +@example +%% +foo return TOK_KEYWORD; +foobar return TOK_KEYWORD; + +fooba | +foob | +fo @{ + /* false alarm, not really a keyword */ + return TOK_ID; + @} +@end example + +Eliminating backing up among a list of keywords can also +be done using a "catch-all" rule: + +@example +%% +foo return TOK_KEYWORD; +foobar return TOK_KEYWORD; + +[a-z]+ return TOK_ID; +@end example + +This is usually the best solution when appropriate. + +Backing up messages tend to cascade. With a complicated +set of rules it's not uncommon to get hundreds of +messages. If one can decipher them, though, it often only +takes a dozen or so rules to eliminate the backing up +(though it's easy to make a mistake and have an error rule +accidentally match a valid token. A possible future @code{flex} +feature will be to automatically add rules to eliminate +backing up). + +It's important to keep in mind that you gain the benefits +of eliminating backing up only if you eliminate @emph{every} +instance of backing up. Leaving just one means you gain +nothing. + +@var{Variable} trailing context (where both the leading and +trailing parts do not have a fixed length) entails almost +the same performance loss as @code{REJECT} (i.e., substantial). +So when possible a rule like: + +@example +%% +mouse|rat/(cat|dog) run(); +@end example + +@noindent +is better written: + +@example +%% +mouse/cat|dog run(); +rat/cat|dog run(); +@end example + +@noindent +or as + +@example +%% +mouse|rat/cat run(); +mouse|rat/dog run(); +@end example + +Note that here the special '|' action does @emph{not} provide any +savings, and can even make things worse (see Deficiencies +/ Bugs below). + +Another area where the user can increase a scanner's +performance (and one that's easier to implement) arises from +the fact that the longer the tokens matched, the faster +the scanner will run. This is because with long tokens +the processing of most input characters takes place in the +(short) inner scanning loop, and does not often have to go +through the additional work of setting up the scanning +environment (e.g., @code{yytext}) for the action. Recall the +scanner for C comments: + +@example +%x comment +%% + int line_num = 1; + +"/*" BEGIN(comment); + +<comment>[^*\n]* +<comment>"*"+[^*/\n]* +<comment>\n ++line_num; +<comment>"*"+"/" BEGIN(INITIAL); +@end example + +This could be sped up by writing it as: + +@example +%x comment +%% + int line_num = 1; + +"/*" BEGIN(comment); + +<comment>[^*\n]* +<comment>[^*\n]*\n ++line_num; +<comment>"*"+[^*/\n]* +<comment>"*"+[^*/\n]*\n ++line_num; +<comment>"*"+"/" BEGIN(INITIAL); +@end example + +Now instead of each newline requiring the processing of +another action, recognizing the newlines is "distributed" +over the other rules to keep the matched text as long as +possible. Note that @emph{adding} rules does @emph{not} slow down the +scanner! The speed of the scanner is independent of the +number of rules or (modulo the considerations given at the +beginning of this section) how complicated the rules are +with regard to operators such as '*' and '|'. + +A final example in speeding up a scanner: suppose you want +to scan through a file containing identifiers and +keywords, one per line and with no other extraneous +characters, and recognize all the keywords. A natural first +approach is: + +@example +%% +asm | +auto | +break | +@dots{} etc @dots{} +volatile | +while /* it's a keyword */ + +.|\n /* it's not a keyword */ +@end example + +To eliminate the back-tracking, introduce a catch-all +rule: + +@example +%% +asm | +auto | +break | +... etc ... +volatile | +while /* it's a keyword */ + +[a-z]+ | +.|\n /* it's not a keyword */ +@end example + +Now, if it's guaranteed that there's exactly one word per +line, then we can reduce the total number of matches by a +half by merging in the recognition of newlines with that +of the other tokens: + +@example +%% +asm\n | +auto\n | +break\n | +@dots{} etc @dots{} +volatile\n | +while\n /* it's a keyword */ + +[a-z]+\n | +.|\n /* it's not a keyword */ +@end example + +One has to be careful here, as we have now reintroduced +backing up into the scanner. In particular, while @emph{we} know +that there will never be any characters in the input +stream other than letters or newlines, @code{flex} can't figure +this out, and it will plan for possibly needing to back up +when it has scanned a token like "auto" and then the next +character is something other than a newline or a letter. +Previously it would then just match the "auto" rule and be +done, but now it has no "auto" rule, only a "auto\n" rule. +To eliminate the possibility of backing up, we could +either duplicate all rules but without final newlines, or, +since we never expect to encounter such an input and +therefore don't how it's classified, we can introduce one +more catch-all rule, this one which doesn't include a +newline: + +@example +%% +asm\n | +auto\n | +break\n | +@dots{} etc @dots{} +volatile\n | +while\n /* it's a keyword */ + +[a-z]+\n | +[a-z]+ | +.|\n /* it's not a keyword */ +@end example + +Compiled with @samp{-Cf}, this is about as fast as one can get a +@code{flex} scanner to go for this particular problem. + +A final note: @code{flex} is slow when matching NUL's, +particularly when a token contains multiple NUL's. It's best to +write rules which match @emph{short} amounts of text if it's +anticipated that the text will often include NUL's. + +Another final note regarding performance: as mentioned +above in the section How the Input is Matched, dynamically +resizing @code{yytext} to accommodate huge tokens is a slow +process because it presently requires that the (huge) token +be rescanned from the beginning. Thus if performance is +vital, you should attempt to match "large" quantities of +text but not "huge" quantities, where the cutoff between +the two is at about 8K characters/token. + +@node C++, Incompatibilities, Performance, Top +@section Generating C++ scanners + +@code{flex} provides two different ways to generate scanners for +use with C++. The first way is to simply compile a +scanner generated by @code{flex} using a C++ compiler instead of a C +compiler. You should not encounter any compilations +errors (please report any you find to the email address +given in the Author section below). You can then use C++ +code in your rule actions instead of C code. Note that +the default input source for your scanner remains @code{yyin}, +and default echoing is still done to @code{yyout}. Both of these +remain @samp{FILE *} variables and not C++ @code{streams}. + +You can also use @code{flex} to generate a C++ scanner class, using +the @samp{-+} option, (or, equivalently, @samp{%option c++}), which +is automatically specified if the name of the flex executable ends +in a @samp{+}, such as @code{flex++}. When using this option, flex +defaults to generating the scanner to the file @file{lex.yy.cc} instead +of @file{lex.yy.c}. The generated scanner includes the header file +@file{FlexLexer.h}, which defines the interface to two C++ classes. + +The first class, @code{FlexLexer}, provides an abstract base +class defining the general scanner class interface. It +provides the following member functions: + +@table @samp +@item const char* YYText() +returns the text of the most recently matched +token, the equivalent of @code{yytext}. + +@item int YYLeng() +returns the length of the most recently matched +token, the equivalent of @code{yyleng}. + +@item int lineno() const +returns the current input line number (see @samp{%option yylineno}), +or 1 if @samp{%option yylineno} was not used. + +@item void set_debug( int flag ) +sets the debugging flag for the scanner, equivalent to assigning to +@code{yy_flex_debug} (see the Options section above). Note that you +must build the scanner using @samp{%option debug} to include debugging +information in it. + +@item int debug() const +returns the current setting of the debugging flag. +@end table + +Also provided are member functions equivalent to +@samp{yy_switch_to_buffer(), yy_create_buffer()} (though the +first argument is an @samp{istream*} object pointer and not a +@samp{FILE*}, @samp{yy_flush_buffer()}, @samp{yy_delete_buffer()}, +and @samp{yyrestart()} (again, the first argument is a @samp{istream*} +object pointer). + +The second class defined in @file{FlexLexer.h} is @code{yyFlexLexer}, +which is derived from @code{FlexLexer}. It defines the following +additional member functions: + +@table @samp +@item yyFlexLexer( istream* arg_yyin = 0, ostream* arg_yyout = 0 ) +constructs a @code{yyFlexLexer} object using the given +streams for input and output. If not specified, +the streams default to @code{cin} and @code{cout}, respectively. + +@item virtual int yylex() +performs the same role is @samp{yylex()} does for ordinary +flex scanners: it scans the input stream, consuming +tokens, until a rule's action returns a value. If you derive a subclass +@var{S} +from @code{yyFlexLexer} +and want to access the member functions and variables of +@var{S} +inside @samp{yylex()}, +then you need to use @samp{%option yyclass="@var{S}"} +to inform @code{flex} +that you will be using that subclass instead of @code{yyFlexLexer}. +In this case, rather than generating @samp{yyFlexLexer::yylex()}, +@code{flex} generates @samp{@var{S}::yylex()} +(and also generates a dummy @samp{yyFlexLexer::yylex()} +that calls @samp{yyFlexLexer::LexerError()} +if called). + +@item virtual void switch_streams(istream* new_in = 0, ostream* new_out = 0) +reassigns @code{yyin} to @code{new_in} +(if non-nil) +and @code{yyout} to @code{new_out} +(ditto), deleting the previous input buffer if @code{yyin} +is reassigned. + +@item int yylex( istream* new_in = 0, ostream* new_out = 0 ) +first switches the input streams via @samp{switch_streams( new_in, new_out )} +and then returns the value of @samp{yylex()}. +@end table + +In addition, @code{yyFlexLexer} defines the following protected +virtual functions which you can redefine in derived +classes to tailor the scanner: + +@table @samp +@item virtual int LexerInput( char* buf, int max_size ) +reads up to @samp{max_size} characters into @var{buf} and +returns the number of characters read. To indicate +end-of-input, return 0 characters. Note that +"interactive" scanners (see the @samp{-B} and @samp{-I} flags) +define the macro @code{YY_INTERACTIVE}. If you redefine +@code{LexerInput()} and need to take different actions +depending on whether or not the scanner might be +scanning an interactive input source, you can test +for the presence of this name via @samp{#ifdef}. + +@item virtual void LexerOutput( const char* buf, int size ) +writes out @var{size} characters from the buffer @var{buf}, +which, while NUL-terminated, may also contain +"internal" NUL's if the scanner's rules can match +text with NUL's in them. + +@item virtual void LexerError( const char* msg ) +reports a fatal error message. The default version +of this function writes the message to the stream +@code{cerr} and exits. +@end table + +Note that a @code{yyFlexLexer} object contains its @emph{entire} +scanning state. Thus you can use such objects to create +reentrant scanners. You can instantiate multiple instances of +the same @code{yyFlexLexer} class, and you can also combine +multiple C++ scanner classes together in the same program +using the @samp{-P} option discussed above. +Finally, note that the @samp{%array} feature is not available to +C++ scanner classes; you must use @samp{%pointer} (the default). + +Here is an example of a simple C++ scanner: + +@example + // An example of using the flex C++ scanner class. + +%@{ +int mylineno = 0; +%@} + +string \"[^\n"]+\" + +ws [ \t]+ + +alpha [A-Za-z] +dig [0-9] +name (@{alpha@}|@{dig@}|\$)(@{alpha@}|@{dig@}|[_.\-/$])* +num1 [-+]?@{dig@}+\.?([eE][-+]?@{dig@}+)? +num2 [-+]?@{dig@}*\.@{dig@}+([eE][-+]?@{dig@}+)? +number @{num1@}|@{num2@} + +%% + +@{ws@} /* skip blanks and tabs */ + +"/*" @{ + int c; + + while((c = yyinput()) != 0) + @{ + if(c == '\n') + ++mylineno; + + else if(c == '*') + @{ + if((c = yyinput()) == '/') + break; + else + unput(c); + @} + @} + @} + +@{number@} cout << "number " << YYText() << '\n'; + +\n mylineno++; + +@{name@} cout << "name " << YYText() << '\n'; + +@{string@} cout << "string " << YYText() << '\n'; + +%% + +Version 2.5 December 1994 44 + +int main( int /* argc */, char** /* argv */ ) + @{ + FlexLexer* lexer = new yyFlexLexer; + while(lexer->yylex() != 0) + ; + return 0; + @} +@end example + +If you want to create multiple (different) lexer classes, +you use the @samp{-P} flag (or the @samp{prefix=} option) to rename each +@code{yyFlexLexer} to some other @code{xxFlexLexer}. You then can +include @samp{<FlexLexer.h>} in your other sources once per lexer +class, first renaming @code{yyFlexLexer} as follows: + +@example +#undef yyFlexLexer +#define yyFlexLexer xxFlexLexer +#include <FlexLexer.h> + +#undef yyFlexLexer +#define yyFlexLexer zzFlexLexer +#include <FlexLexer.h> +@end example + +if, for example, you used @samp{%option prefix="xx"} for one of +your scanners and @samp{%option prefix="zz"} for the other. + +IMPORTANT: the present form of the scanning class is +@emph{experimental} and may change considerably between major +releases. + +@node Incompatibilities, Diagnostics, C++, Top +@section Incompatibilities with @code{lex} and POSIX + +@code{flex} is a rewrite of the AT&T Unix @code{lex} tool (the two +implementations do not share any code, though), with some +extensions and incompatibilities, both of which are of +concern to those who wish to write scanners acceptable to +either implementation. Flex is fully compliant with the +POSIX @code{lex} specification, except that when using @samp{%pointer} +(the default), a call to @samp{unput()} destroys the contents of +@code{yytext}, which is counter to the POSIX specification. + +In this section we discuss all of the known areas of +incompatibility between flex, AT&T lex, and the POSIX +specification. + +@code{flex's} @samp{-l} option turns on maximum compatibility with the +original AT&T @code{lex} implementation, at the cost of a major +loss in the generated scanner's performance. We note +below which incompatibilities can be overcome using the @samp{-l} +option. + +@code{flex} is fully compatible with @code{lex} with the following +exceptions: + +@itemize - +@item +The undocumented @code{lex} scanner internal variable @code{yylineno} +is not supported unless @samp{-l} or @samp{%option yylineno} is used. +@code{yylineno} should be maintained on a per-buffer basis, rather +than a per-scanner (single global variable) basis. @code{yylineno} is +not part of the POSIX specification. + +@item +The @samp{input()} routine is not redefinable, though it +may be called to read characters following whatever +has been matched by a rule. If @samp{input()} encounters +an end-of-file the normal @samp{yywrap()} processing is +done. A ``real'' end-of-file is returned by +@samp{input()} as @code{EOF}. + +Input is instead controlled by defining the +@code{YY_INPUT} macro. + +The @code{flex} restriction that @samp{input()} cannot be +redefined is in accordance with the POSIX +specification, which simply does not specify any way of +controlling the scanner's input other than by making +an initial assignment to @code{yyin}. + +@item +The @samp{unput()} routine is not redefinable. This +restriction is in accordance with POSIX. + +@item +@code{flex} scanners are not as reentrant as @code{lex} scanners. +In particular, if you have an interactive scanner +and an interrupt handler which long-jumps out of +the scanner, and the scanner is subsequently called +again, you may get the following message: + +@example +fatal flex scanner internal error--end of buffer missed +@end example + +To reenter the scanner, first use + +@example +yyrestart( yyin ); +@end example + +Note that this call will throw away any buffered +input; usually this isn't a problem with an +interactive scanner. + +Also note that flex C++ scanner classes @emph{are} +reentrant, so if using C++ is an option for you, you +should use them instead. See "Generating C++ +Scanners" above for details. + +@item +@samp{output()} is not supported. Output from the @samp{ECHO} +macro is done to the file-pointer @code{yyout} (default +@code{stdout}). + +@samp{output()} is not part of the POSIX specification. + +@item +@code{lex} does not support exclusive start conditions +(%x), though they are in the POSIX specification. + +@item +When definitions are expanded, @code{flex} encloses them +in parentheses. With lex, the following: + +@example +NAME [A-Z][A-Z0-9]* +%% +foo@{NAME@}? printf( "Found it\n" ); +%% +@end example + +will not match the string "foo" because when the +macro is expanded the rule is equivalent to +"foo[A-Z][A-Z0-9]*?" and the precedence is such that the +'?' is associated with "[A-Z0-9]*". With @code{flex}, the +rule will be expanded to "foo([A-Z][A-Z0-9]*)?" and +so the string "foo" will match. + +Note that if the definition begins with @samp{^} or ends +with @samp{$} then it is @emph{not} expanded with parentheses, to +allow these operators to appear in definitions +without losing their special meanings. But the +@samp{<s>, /}, and @samp{<<EOF>>} operators cannot be used in a +@code{flex} definition. + +Using @samp{-l} results in the @code{lex} behavior of no +parentheses around the definition. + +The POSIX specification is that the definition be enclosed in +parentheses. + +@item +Some implementations of @code{lex} allow a rule's action to begin on +a separate line, if the rule's pattern has trailing whitespace: + +@example +%% +foo|bar<space here> + @{ foobar_action(); @} +@end example + +@code{flex} does not support this feature. + +@item +The @code{lex} @samp{%r} (generate a Ratfor scanner) option is +not supported. It is not part of the POSIX +specification. + +@item +After a call to @samp{unput()}, @code{yytext} is undefined until +the next token is matched, unless the scanner was +built using @samp{%array}. This is not the case with @code{lex} +or the POSIX specification. The @samp{-l} option does +away with this incompatibility. + +@item +The precedence of the @samp{@{@}} (numeric range) operator +is different. @code{lex} interprets "abc@{1,3@}" as "match +one, two, or three occurrences of 'abc'", whereas +@code{flex} interprets it as "match 'ab' followed by one, +two, or three occurrences of 'c'". The latter is +in agreement with the POSIX specification. + +@item +The precedence of the @samp{^} operator is different. @code{lex} +interprets "^foo|bar" as "match either 'foo' at the +beginning of a line, or 'bar' anywhere", whereas +@code{flex} interprets it as "match either 'foo' or 'bar' +if they come at the beginning of a line". The +latter is in agreement with the POSIX specification. + +@item +The special table-size declarations such as @samp{%a} +supported by @code{lex} are not required by @code{flex} scanners; +@code{flex} ignores them. + +@item +The name FLEX_SCANNER is #define'd so scanners may +be written for use with either @code{flex} or @code{lex}. +Scanners also include @code{YY_FLEX_MAJOR_VERSION} and +@code{YY_FLEX_MINOR_VERSION} indicating which version of +@code{flex} generated the scanner (for example, for the +2.5 release, these defines would be 2 and 5 +respectively). +@end itemize + +The following @code{flex} features are not included in @code{lex} or the +POSIX specification: + +@example +C++ scanners +%option +start condition scopes +start condition stacks +interactive/non-interactive scanners +yy_scan_string() and friends +yyterminate() +yy_set_interactive() +yy_set_bol() +YY_AT_BOL() +<<EOF>> +<*> +YY_DECL +YY_START +YY_USER_ACTION +YY_USER_INIT +#line directives +%@{@}'s around actions +multiple actions on a line +@end example + +@noindent +plus almost all of the flex flags. The last feature in +the list refers to the fact that with @code{flex} you can put +multiple actions on the same line, separated with +semicolons, while with @code{lex}, the following + +@example +foo handle_foo(); ++num_foos_seen; +@end example + +@noindent +is (rather surprisingly) truncated to + +@example +foo handle_foo(); +@end example + +@code{flex} does not truncate the action. Actions that are not +enclosed in braces are simply terminated at the end of the +line. + +@node Diagnostics, Files, Incompatibilities, Top +@section Diagnostics + +@table @samp +@item warning, rule cannot be matched +indicates that the given +rule cannot be matched because it follows other rules that +will always match the same text as it. For example, in +the following "foo" cannot be matched because it comes +after an identifier "catch-all" rule: + +@example +[a-z]+ got_identifier(); +foo got_foo(); +@end example + +Using @code{REJECT} in a scanner suppresses this warning. + +@item warning, -s option given but default rule can be matched +means that it is possible (perhaps only in a particular +start condition) that the default rule (match any single +character) is the only one that will match a particular +input. Since @samp{-s} was given, presumably this is not +intended. + +@item reject_used_but_not_detected undefined +@itemx yymore_used_but_not_detected undefined +These errors can +occur at compile time. They indicate that the scanner +uses @code{REJECT} or @samp{yymore()} but that @code{flex} failed to notice the +fact, meaning that @code{flex} scanned the first two sections +looking for occurrences of these actions and failed to +find any, but somehow you snuck some in (via a #include +file, for example). Use @samp{%option reject} or @samp{%option yymore} +to indicate to flex that you really do use these features. + +@item flex scanner jammed +a scanner compiled with @samp{-s} has +encountered an input string which wasn't matched by any of +its rules. This error can also occur due to internal +problems. + +@item token too large, exceeds YYLMAX +your scanner uses @samp{%array} +and one of its rules matched a string longer than the @samp{YYL-} +@code{MAX} constant (8K bytes by default). You can increase the +value by #define'ing @code{YYLMAX} in the definitions section of +your @code{flex} input. + +@item scanner requires -8 flag to use the character '@var{x}' +Your +scanner specification includes recognizing the 8-bit +character @var{x} and you did not specify the -8 flag, and your +scanner defaulted to 7-bit because you used the @samp{-Cf} or @samp{-CF} +table compression options. See the discussion of the @samp{-7} +flag for details. + +@item flex scanner push-back overflow +you used @samp{unput()} to push +back so much text that the scanner's buffer could not hold +both the pushed-back text and the current token in @code{yytext}. +Ideally the scanner should dynamically resize the buffer +in this case, but at present it does not. + +@item input buffer overflow, can't enlarge buffer because scanner uses REJECT +the scanner was working on matching an +extremely large token and needed to expand the input +buffer. This doesn't work with scanners that use @code{REJECT}. + +@item fatal flex scanner internal error--end of buffer missed +This can occur in an scanner which is reentered after a +long-jump has jumped out (or over) the scanner's +activation frame. Before reentering the scanner, use: + +@example +yyrestart( yyin ); +@end example + +@noindent +or, as noted above, switch to using the C++ scanner class. + +@item too many start conditions in <> construct! +you listed +more start conditions in a <> construct than exist (so you +must have listed at least one of them twice). +@end table + +@node Files, Deficiencies, Diagnostics, Top +@section Files + +@table @file +@item -lfl +library with which scanners must be linked. + +@item lex.yy.c +generated scanner (called @file{lexyy.c} on some systems). + +@item lex.yy.cc +generated C++ scanner class, when using @samp{-+}. + +@item <FlexLexer.h> +header file defining the C++ scanner base class, +@code{FlexLexer}, and its derived class, @code{yyFlexLexer}. + +@item flex.skl +skeleton scanner. This file is only used when +building flex, not when flex executes. + +@item lex.backup +backing-up information for @samp{-b} flag (called @file{lex.bck} +on some systems). +@end table + +@node Deficiencies, See also, Files, Top +@section Deficiencies / Bugs + +Some trailing context patterns cannot be properly matched +and generate warning messages ("dangerous trailing +context"). These are patterns where the ending of the first +part of the rule matches the beginning of the second part, +such as "zx*/xy*", where the 'x*' matches the 'x' at the +beginning of the trailing context. (Note that the POSIX +draft states that the text matched by such patterns is +undefined.) + +For some trailing context rules, parts which are actually +fixed-length are not recognized as such, leading to the +abovementioned performance loss. In particular, parts +using '|' or @{n@} (such as "foo@{3@}") are always considered +variable-length. + +Combining trailing context with the special '|' action can +result in @emph{fixed} trailing context being turned into the +more expensive @var{variable} trailing context. For example, in +the following: + +@example +%% +abc | +xyz/def +@end example + +Use of @samp{unput()} invalidates yytext and yyleng, unless the +@samp{%array} directive or the @samp{-l} option has been used. + +Pattern-matching of NUL's is substantially slower than +matching other characters. + +Dynamic resizing of the input buffer is slow, as it +entails rescanning all the text matched so far by the +current (generally huge) token. + +Due to both buffering of input and read-ahead, you cannot +intermix calls to <stdio.h> routines, such as, for +example, @samp{getchar()}, with @code{flex} rules and expect it to work. +Call @samp{input()} instead. + +The total table entries listed by the @samp{-v} flag excludes the +number of table entries needed to determine what rule has +been matched. The number of entries is equal to the +number of DFA states if the scanner does not use @code{REJECT}, and +somewhat greater than the number of states if it does. + +@code{REJECT} cannot be used with the @samp{-f} or @samp{-F} options. + +The @code{flex} internal algorithms need documentation. + +@node See also, Author, Deficiencies, Top +@section See also + +@code{lex}(1), @code{yacc}(1), @code{sed}(1), @code{awk}(1). + +John Levine, Tony Mason, and Doug Brown: Lex & Yacc; +O'Reilly and Associates. Be sure to get the 2nd edition. + +M. E. Lesk and E. Schmidt, LEX - Lexical Analyzer Generator. + +Alfred Aho, Ravi Sethi and Jeffrey Ullman: Compilers: +Principles, Techniques and Tools; Addison-Wesley (1986). +Describes the pattern-matching techniques used by @code{flex} +(deterministic finite automata). + +@node Author, , See also, Top +@section Author + +Vern Paxson, with the help of many ideas and much inspiration from +Van Jacobson. Original version by Jef Poskanzer. The fast table +representation is a partial implementation of a design done by Van +Jacobson. The implementation was done by Kevin Gong and Vern Paxson. + +Thanks to the many @code{flex} beta-testers, feedbackers, and +contributors, especially Francois Pinard, Casey Leedom, Stan +Adermann, Terry Allen, David Barker-Plummer, John Basrai, Nelson +H.F. Beebe, @samp{benson@@odi.com}, Karl Berry, Peter A. Bigot, +Simon Blanchard, Keith Bostic, Frederic Brehm, Ian Brockbank, Kin +Cho, Nick Christopher, Brian Clapper, J.T. Conklin, Jason Coughlin, +Bill Cox, Nick Cropper, Dave Curtis, Scott David Daniels, Chris +G. Demetriou, Theo Deraadt, Mike Donahue, Chuck Doucette, Tom Epperly, +Leo Eskin, Chris Faylor, Chris Flatters, Jon Forrest, Joe Gayda, Kaveh +R. Ghazi, Eric Goldman, Christopher M. Gould, Ulrich Grepel, Peer +Griebel, Jan Hajic, Charles Hemphill, NORO Hideo, Jarkko Hietaniemi, +Scott Hofmann, Jeff Honig, Dana Hudes, Eric Hughes, John Interrante, +Ceriel Jacobs, Michal Jaegermann, Sakari Jalovaara, Jeffrey R. Jones, +Henry Juengst, Klaus Kaempf, Jonathan I. Kamens, Terrence O Kane, +Amir Katz, @samp{ken@@ken.hilco.com}, Kevin B. Kenny, Steve Kirsch, +Winfried Koenig, Marq Kole, Ronald Lamprecht, Greg Lee, Rohan Lenard, +Craig Leres, John Levine, Steve Liddle, Mike Long, Mohamed el Lozy, +Brian Madsen, Malte, Joe Marshall, Bengt Martensson, Chris Metcalf, +Luke Mewburn, Jim Meyering, R. Alexander Milowski, Erik Naggum, +G.T. Nicol, Landon Noll, James Nordby, Marc Nozell, Richard Ohnemus, +Karsten Pahnke, Sven Panne, Roland Pesch, Walter Pelissero, Gaumond +Pierre, Esmond Pitt, Jef Poskanzer, Joe Rahmeh, Jarmo Raiha, Frederic +Raimbault, Pat Rankin, Rick Richardson, Kevin Rodgers, Kai Uwe Rommel, +Jim Roskind, Alberto Santini, Andreas Scherer, Darrell Schiebel, Raf +Schietekat, Doug Schmidt, Philippe Schnoebelen, Andreas Schwab, Alex +Siegel, Eckehard Stolz, Jan-Erik Strvmquist, Mike Stump, Paul Stuart, +Dave Tallman, Ian Lance Taylor, Chris Thewalt, Richard M. Timoney, +Jodi Tsai, Paul Tuinenga, Gary Weik, Frank Whaley, Gerhard Wilhelms, +Kent Williams, Ken Yap, Ron Zellar, Nathan Zelle, David Zuhn, and +those whose names have slipped my marginal mail-archiving skills but +whose contributions are appreciated all the same. + +Thanks to Keith Bostic, Jon Forrest, Noah Friedman, John Gilmore, +Craig Leres, John Levine, Bob Mulcahy, G.T. Nicol, Francois Pinard, +Rich Salz, and Richard Stallman for help with various distribution +headaches. + +Thanks to Esmond Pitt and Earle Horton for 8-bit character support; +to Benson Margulies and Fred Burke for C++ support; to Kent Williams +and Tom Epperly for C++ class support; to Ove Ewerlid for support of +NUL's; and to Eric Hughes for support of multiple buffers. + +This work was primarily done when I was with the Real Time Systems +Group at the Lawrence Berkeley Laboratory in Berkeley, CA. Many thanks +to all there for the support I received. + +Send comments to @samp{vern@@ee.lbl.gov}. + +@c @node Index, , Top, Top +@c @unnumbered Index +@c +@c @printindex cp + +@contents +@bye + +@c Local variables: +@c texinfo-column-for-description: 32 +@c End: |