diff options
Diffstat (limited to 'examples')
36 files changed, 2221 insertions, 0 deletions
diff --git a/examples/README b/examples/README new file mode 100644 index 0000000..46f542b --- /dev/null +++ b/examples/README @@ -0,0 +1,15 @@ +This directory contains some examples of what you can do with +flex. These files are not tested regularly so you might have to tinker +a bit before they work for you. Updates, new files and patches are welcome. + + - debflex.awk, an awk script for anotating flex debug output. + It presently only works with gawk and mawk, not with "old" + or "new" awk. + + - testxxLexer.l, a sample C++ program that uses flex's scanner + class option ("-+"). + + - fastwc/, a subdirectory containing examples of how to use flex + to write progressively higher-performance versions of the Unix + "wc" utility. This certainly should work with 2.5, but hasn't + been tested. diff --git a/examples/debflex.awk b/examples/debflex.awk new file mode 100644 index 0000000..b1eda49 --- /dev/null +++ b/examples/debflex.awk @@ -0,0 +1,119 @@ +# Clarify the flex debug trace by substituting first line of each rule. +# Francois Pinard <pinard@iro.umontreal.ca>, July 1990. +# +# Rewritten to process correctly \n's in scanner input. +# BEGIN section modified to correct a collection of rules. +# Michal Jaegermann <michal@phys.ualberta.ca>, December 1993 +# +# Sample usage: +# flex -d PROGRAM.l +# gcc -o PROGRAM PROGRAM.c -lfl +# PROGRAM 2>&1 | gawk -f debflex.awk PROGRAM.l +# +# (VP's note: this script presently does not work with either "old" or +# "new" awk; fixes so it does will be welcome) + +BEGIN { + # Insure proper usage. + + if (ARGC != 2) { + print "usage: gawk -f debflex.awk FLEX_SOURCE <DEBUG_OUTPUT"; + exit (1); + } + + # Remove and save the name of flex source. + + source = ARGV[1]; + ARGC--; + + # Swallow the flex source file. + + line = 0; + section = 1; + while (getline <source) { + + # Count the lines. + + line++; + + # Count the sections. When encountering section 3, + # break out of the awk BEGIN block. + + if (match ($0, /^%%/)) { + section++; + if (section == 3) { + break; + } + } + else { + # Only the lines in section 2 which do not begin in a + # tab or space might be referred to by the flex debug + # trace. Save only those lines. + + if (section == 2 && match ($0, /^[^ \t]/)) { + rules[line] = $0; + } + } + } + dashes = "-----------------------------------------------------------"; + collect = ""; + line = 0; +} + +# collect complete rule output from a scanner +$0 !~ /^--/ { + collect = collect "\n" $0; + next; +} +# otherwise we have a new rule - process what we got so far +{ + process(); +} +# and the same thing if we hit EOF +END { + process(); +} + +function process() { + + # splitting this way we loose some double dashes and + # left parentheses from echoed input - a small price to pay + n = split(collect, field, "\n--|[(]"); + + # this loop kicks in only when we already collected something + for (i = 1; i <= n; i++) { + if (0 != line) { + # we do not care for traces of newlines. + if (0 == match(field[i], /\"\n+\"[)]/)) { + if (rules[line]) { + text = field[i]; + while ( ++i <= n) { + text = text field[i]; + } + printf("%s:%d: %-8s -- %s\n", + source, line, text, rules[line]); + } + else { + print; + printf "%s:%d: *** No such rule.\n", source, line; + } + } + line = 0; + break; + } + if ("" != field[i]) { + if ("end of buffer or a NUL)" == field[i]) { + print dashes; # Simplify trace of buffer reloads + continue; + } + if (match(field[i], /accepting rule at line /)) { + # force interpretation of line as a number + line = 0 + substr(field[i], RLENGTH); + continue; + } + # echo everything else + printf("--%s\n", field[i]); + } + } + collect = "\n" $0; # ... and start next trace +} diff --git a/examples/manual/ChangeLog b/examples/manual/ChangeLog new file mode 100644 index 0000000..fb5e4a2 --- /dev/null +++ b/examples/manual/ChangeLog @@ -0,0 +1,24 @@ +Tue Oct 5 21:51:59 1993 Vern Paxson + + * Removed FILTER/ subdirectory. + + * Removed alloca.c. + + * Changed Makefile definition of CC to just "gcc -g", removed + assumption of alloca being present. + + * Added pointer to MISC/fastwc/ to wc.lex. + +Tue Jun 8 15:47:39 1993 Gavin Thomas Nicol (nick at sillybugs) + + * Changed main() in wc.lex extensively. The old version would not + work correctly without the YY_NEW_FILE. (lex handles the older + version OK though). + + * Added a rule to expr.lex to handle whitespace. The old version + reported an illegal character. + + * Added -traditional to the gcc flags because the flex definition + for free() clashes with some systems that have old header files. + + diff --git a/examples/manual/Makefile b/examples/manual/Makefile new file mode 100644 index 0000000..f4d8297 --- /dev/null +++ b/examples/manual/Makefile @@ -0,0 +1,88 @@ +############################################################# +# +# Makefile : Makefile for Flex examples. +# Author : G.T.Nicol +# Last Updated : 1993/10/05 +# +# If you use bison, you may have to supply an alloca +# +############################################################# + +CC = gcc -g +LEX = flex -i -I +YACC = bison -d -y +ALLOCA = + +############################################################ +# +# DO NOT CHANGE ANYTHING FROM HERE ON !!!!!!!!! +# +############################################################ + +all: expr front myname eof wc replace user_act string1\ + string2 yymore numbers dates cat + +expr: expr.y expr.lex + $(YACC) expr.y + $(LEX) expr.lex + $(CC) -o expr lex.yy.c y.tab.c $(ALLOCA) -ll -lm + +front: front.y front.lex + $(YACC) front.y + $(LEX) front.lex + $(CC) -o front lex.yy.c y.tab.c $(ALLOCA) -ll -lm + +numbers: numbers.lex + $(LEX) numbers.lex + $(CC) lex.yy.c -o numbers + +dates: dates.lex + $(LEX) dates.lex + $(CC) lex.yy.c -o dates -ll + +yymore: yymore.lex + $(LEX) yymore.lex + $(CC) lex.yy.c -o yymore -ll + +string1: string1.lex + $(LEX) string1.lex + $(CC) lex.yy.c -o string1 -ll + +string2: string2.lex + $(LEX) string2.lex + $(CC) lex.yy.c -o string2 -ll + +myname: myname.lex + $(LEX) myname.lex + $(CC) lex.yy.c -o myname -ll + +myname2: myname2.lex + $(LEX) myname2.lex + $(CC) lex.yy.c -o myname2 -ll + +eof: eof_rules.lex + $(LEX) eof_rules.lex + $(CC) lex.yy.c -o eof -ll + +wc: wc.lex + $(LEX) wc.lex + $(CC) lex.yy.c -o wc -ll + +cat: cat.lex + $(LEX) cat.lex + $(CC) lex.yy.c -o cat -ll + +replace: replace.lex + $(LEX) replace.lex + $(CC) lex.yy.c -o replace -ll + +user_act: expr.y expr.lex + $(LEX) user_act.lex + $(CC) -o user_act lex.yy.c -ll + +clean: + rm -f *.BAK *.o core *~* *.a + rm -f *.tab.h *.tab.c + rm -f myname expr lex.yy.c *.out eof wc yymore + rm -f replace front user_act string1 string2 + rm -f dates numbers cat diff --git a/examples/manual/README b/examples/manual/README new file mode 100644 index 0000000..860f300 --- /dev/null +++ b/examples/manual/README @@ -0,0 +1,17 @@ +This directory contains the example programs from the manual, and a few +other things as well. To make all the programs, simply type "make", +and assuming you have flex and gcc, all will be well. + +To build the programs individually, type + + make program_name + +For example: + + make expr + + +The subdirectory FILTER contains a collection of the silly filters +that have appeared on the Internet over the years. The author of the +flex manual has included them for fun, but does not guarantee they will +work with flex, or even work at all. diff --git a/examples/manual/cat.lex b/examples/manual/cat.lex new file mode 100644 index 0000000..7890aa2 --- /dev/null +++ b/examples/manual/cat.lex @@ -0,0 +1,45 @@ +/* + * cat.lex: A demonstration of YY_NEW_FILE. + */ + +%{ +#include <stdio.h> + +char **names = NULL; +int current = 1; +%} + +%% +<<EOF>> { + current += 1; + if(names[current] != NULL){ + yyin = fopen(names[current],"r"); + if(yyin == NULL){ + fprintf(stderr,"cat: unable to open %s\n", + names[current]); + yyterminate(); + } + YY_NEW_FILE; + } else { + yyterminate(); + } + } +%% + +int main(int argc, char **argv) +{ + if(argc < 2){ + fprintf(stderr,"Usage: cat files....\n"); + exit(1); + } + names = argv; + + yyin = fopen(names[current],"r"); + if(yyin == NULL){ + fprintf(stderr,"cat: unable to open %s\n", + names[current]); + yyterminate(); + } + + yylex(); +} diff --git a/examples/manual/dates.lex b/examples/manual/dates.lex new file mode 100644 index 0000000..9429e1d --- /dev/null +++ b/examples/manual/dates.lex @@ -0,0 +1,106 @@ +/* + * dates.lex: An example of using start states to + * distinguish between different date formats. + */ + +%{ +#include <ctype.h> + +char month[20],dow[20],day[20],year[20]; + +%} + +skip of|the|[ \t,]* + +mon (mon(day)?) +tue (tue(sday)?) +wed (wed(nesday)?) +thu (thu(rsday)?) +fri (fri(day)?) +sat (sat(urday)?) +sun (sun(day)?) + +day_of_the_week ({mon}|{tue}|{wed}|{thu}|{fri}|{sat}|{sun}) + +jan (jan(uary)?) +feb (feb(ruary)?) +mar (mar(ch)?) +apr (apr(il)?) +may (may) +jun (jun(e)?) +jul (jul(y)?) +aug (aug(ust)?) +sep (sep(tember)?) +oct (oct(ober)?) +nov (nov(ember)?) +dec (dec(ember)?) + +first_half ({jan}|{feb}|{mar}|{apr}|{may}|{jun}) +second_half ({jul}|{aug}|{sep}|{oct}|{nov}|{dec}) +month {first_half}|{second_half} + +nday [1-9]|[1-2][0-9]|3[0-1] +nmonth [1-9]|1[0-2] +nyear [0-9]{1,4} + +year_ext (ad|AD|bc|BC)? +day_ext (st|nd|rd|th)? + +%s LONG SHORT +%s DAY DAY_FIRST YEAR_FIRST YEAR_LAST YFMONTH YLMONTH + +%% + + /* the default is month-day-year */ + +<LONG>{day_of_the_week} strcpy(dow,yytext); +<LONG>{month} strcpy(month,yytext); BEGIN(DAY); + + /* handle the form: day-month-year */ + +<LONG>{nday}{day_ext} strcpy(day,yytext); BEGIN(DAY_FIRST); +<DAY_FIRST>{month} strcpy(month,yytext); BEGIN(LONG); +<DAY>{nday}{day_ext} strcpy(day,yytext); BEGIN(LONG); + +<LONG>{nyear}{year_ext} { + printf("Long:\n"); + printf(" DOW : %s \n",dow); + printf(" Day : %s \n",day); + printf(" Month : %s \n",month); + printf(" Year : %s \n",yytext); + strcpy(dow,""); + strcpy(day,""); + strcpy(month,""); + } + + /* handle dates of the form: day-month-year */ + +<SHORT>{nday} strcpy(day,yytext); BEGIN(YEAR_LAST); +<YEAR_LAST>{nmonth} strcpy(month,yytext);BEGIN(YLMONTH); +<YLMONTH>{nyear} strcpy(year,yytext); BEGIN(SHORT); + + /* handle dates of the form: year-month-day */ + +<SHORT>{nyear} strcpy(year,yytext); BEGIN(YEAR_FIRST); +<YEAR_FIRST>{nmonth} strcpy(month,yytext);BEGIN(YFMONTH); +<YFMONTH>{nday} strcpy(day,yytext); BEGIN(SHORT); + + +<SHORT>\n { + printf("Short:\n"); + printf(" Day : %s \n",day); + printf(" Month : %s \n",month); + printf(" Year : %s \n",year); + strcpy(year,""); + strcpy(day,""); + strcpy(month,""); + } + +long\n BEGIN(LONG); +short\n BEGIN(SHORT); + +{skip}* +\n +. + + diff --git a/examples/manual/datetest.dat b/examples/manual/datetest.dat new file mode 100644 index 0000000..427f407 --- /dev/null +++ b/examples/manual/datetest.dat @@ -0,0 +1,28 @@ +short +1989:12:23 +1989:11:12 +23:12:1989 +11:12:1989 +1989/12/23 +1989/11/12 +23/12/1989 +11/12/1989 +1989-12-23 +1989-11-12 +23-12-1989 +11-12-1989 +long +Friday the 5th of January, 1989 +Friday, 5th of January, 1989 +Friday, January 5th, 1989 +Fri, January 5th, 1989 +Fri, Jan 5th, 1989 +Fri, Jan 5, 1989 +FriJan 5, 1989 +FriJan5, 1989 +FriJan51989 +Jan51989 + + + + diff --git a/examples/manual/eof_rules.lex b/examples/manual/eof_rules.lex new file mode 100644 index 0000000..b575f2c --- /dev/null +++ b/examples/manual/eof_rules.lex @@ -0,0 +1,65 @@ +/* + * eof_rules.lex : An example of using multiple buffers + * EOF rules, and start states + */ + +%{ + +#define MAX_NEST 10 + +YY_BUFFER_STATE include_stack[MAX_NEST]; +int include_count = -1; + +%} + + +%x INCLUDE + +%% + +^"#include"[ \t]*\" BEGIN(INCLUDE); +<INCLUDE>\" BEGIN(INITIAL); +<INCLUDE>[^\"]+ { /* get the include file name */ + if ( include_count >= MAX_NEST){ + fprintf( stderr, "Too many include files" ); + exit( 1 ); + } + + include_stack[++include_count] = YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + if ( ! yyin ){ + fprintf( stderr, "Unable to open \"%s\"\n",yytext); + exit( 1 ); + } + + yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE)); + + BEGIN(INITIAL); + } +<INCLUDE><<EOF>> + { + fprintf( stderr, "EOF in include" ); + yyterminate(); + } +<<EOF>> { + if ( include_count <= 0 ){ + yyterminate(); + } else { + yy_delete_buffer(include_stack[include_count--] ); + yy_switch_to_buffer(include_stack[include_count] ); + BEGIN(INCLUDE); + } + } +[a-z]+ ECHO; +.|\n ECHO; + + + + + + + + + + diff --git a/examples/manual/eof_test01.txt b/examples/manual/eof_test01.txt new file mode 100644 index 0000000..ec5f083 --- /dev/null +++ b/examples/manual/eof_test01.txt @@ -0,0 +1,17 @@ +This is test file #1 + +------------------------------------------------- + +We will now include test #2 in a standard way. + +#include "eof_test02.txt" + +------------------------------------------------- + +And now we will include test # 2 with a different +format + +#include "eof_test02.txt" +------------------------------------------------- + +and this is the end of the test. diff --git a/examples/manual/eof_test02.txt b/examples/manual/eof_test02.txt new file mode 100644 index 0000000..19f52cf --- /dev/null +++ b/examples/manual/eof_test02.txt @@ -0,0 +1,8 @@ +INCLUDE #2 + +This is the second file that will +be included. + +>>> Foo are GNU? + +#include "eof_test03.txt" diff --git a/examples/manual/eof_test03.txt b/examples/manual/eof_test03.txt new file mode 100644 index 0000000..e737879 --- /dev/null +++ b/examples/manual/eof_test03.txt @@ -0,0 +1,7 @@ +INCLUDE #3 + +This is the third file that will +be included. + +>>> echo "I am `whoami`!!" + diff --git a/examples/manual/expr.lex b/examples/manual/expr.lex new file mode 100644 index 0000000..9adfcaa --- /dev/null +++ b/examples/manual/expr.lex @@ -0,0 +1,35 @@ +/* + * expr.lex : Scanner for a simple + * expression parser. + */ + +%{ +#include "y.tab.h" + +%} + +%% + +[0-9]+ { yylval.val = atol(yytext); + return(NUMBER); + } +[0-9]+\.[0-9]+ { + sscanf(yytext,"%f",&yylval.val); + return(NUMBER); + } +"+" return(PLUS); +"-" return(MINUS); +"*" return(MULT); +"/" return(DIV); +"^" return(EXPON); +"(" return(LB); +")" return(RB); +\n return(EOL); +[\t ]* /* throw away whitespace */ +. { yyerror("Illegal character"); + return(EOL); + } +%% + + + diff --git a/examples/manual/expr.y b/examples/manual/expr.y new file mode 100644 index 0000000..1ac3092 --- /dev/null +++ b/examples/manual/expr.y @@ -0,0 +1,64 @@ +/* + * expr.y : A simple yacc expression parser + * Based on the Bison manual example. + */ + +%{ +#include <stdio.h> +#include <math.h> + +%} + +%union { + float val; +} + +%token NUMBER +%token PLUS MINUS MULT DIV EXPON +%token EOL +%token LB RB + +%left MINUS PLUS +%left MULT DIV +%right EXPON + +%type <val> exp NUMBER + +%% +input : + | input line + ; + +line : EOL + | exp EOL { printf("%g\n",$1);} + +exp : NUMBER { $$ = $1; } + | exp PLUS exp { $$ = $1 + $3; } + | exp MINUS exp { $$ = $1 - $3; } + | exp MULT exp { $$ = $1 * $3; } + | exp DIV exp { $$ = $1 / $3; } + | MINUS exp %prec MINUS { $$ = -$2; } + | exp EXPON exp { $$ = pow($1,$3);} + | LB exp RB { $$ = $2; } + ; + +%% + +yyerror(char *message) +{ + printf("%s\n",message); +} + +int main(int argc, char *argv[]) +{ + yyparse(); + return(0); +} + + + + + + + + diff --git a/examples/manual/front.lex b/examples/manual/front.lex new file mode 100644 index 0000000..449cb00 --- /dev/null +++ b/examples/manual/front.lex @@ -0,0 +1,40 @@ +%{ +#include <stdio.h> +#include <string.h> +#include "y.tab.h" /* this comes from bison */ + +#define TRUE 1 +#define FALSE 0 + +#define copy_and_return(token_type) { strcpy(yylval.name,yytext); \ + return(token_type); } + +int yylexlinenum = 0; /* so we can count lines */ +%} + +%% + /* Lexical scanning rules begin from here. */ + +MEN|WOMEN|STOCKS|TREES copy_and_return(NOUN) +MISTAKES|GNUS|EMPLOYEES copy_and_return(NOUN) +LOSERS|USERS|CARS|WINDOWS copy_and_return(NOUN) + +DATABASE|NETWORK|FSF|GNU copy_and_return(PROPER_NOUN) +COMPANY|HOUSE|OFFICE|LPF copy_and_return(PROPER_NOUN) + +THE|THIS|THAT|THOSE copy_and_return(DECLARATIVE) + +ALL|FIRST|LAST copy_and_return(CONDITIONAL) + +FIND|SEARCH|SORT|ERASE|KILL copy_and_return(VERB) +ADD|REMOVE|DELETE|PRINT copy_and_return(VERB) + +QUICKLY|SLOWLY|CAREFULLY copy_and_return(ADVERB) + +IN|AT|ON|AROUND|INSIDE|ON copy_and_return(POSITIONAL) + +"." return(PERIOD); +"\n" yylexlinenum++; return(NEWLINE); +. +%% + diff --git a/examples/manual/front.y b/examples/manual/front.y new file mode 100644 index 0000000..77b5ca1 --- /dev/null +++ b/examples/manual/front.y @@ -0,0 +1,118 @@ +/* C code supplied at the beginning of the file. */ + +%{ + +#include <stdio.h> +#include <string.h> + +extern int yylexlinenum; /* these are in YYlex */ +extern char *yytext; /* current token */ + + +%} + +/* Keywords and reserved words begin here. */ + +%union{ /* this is the data union */ + char name[128]; /* names */ +} + +/*-------------------- the reserved words -----------------------------*/ + +%token PERIOD +%token NEWLINE +%token POSITIONAL + +%token VERB +%token ADVERB + +%token PROPER_NOUN +%token NOUN + +%token DECLARATIVE +%token CONDITIONAL + + +%type <name> declarative +%type <name> verb_phrase +%type <name> noun_phrase +%type <name> position_phrase +%type <name> adverb + +%type <name> POSITIONAL VERB ADVERB PROPER_NOUN +%type <name> NOUN DECLARATIVE CONDITIONAL + +%% + +sentence_list : sentence + | sentence_list NEWLINE sentence + ; + + +sentence : verb_phrase noun_phrase position_phrase adverb period + { + printf("I understand that sentence.\n"); + printf("VP = %s \n",$1); + printf("NP = %s \n",$2); + printf("PP = %s \n",$3); + printf("AD = %s \n",$4); + } + | { yyerror("That's a strange sentence !!"); } + ; + +position_phrase : POSITIONAL declarative PROPER_NOUN + { + sprintf($$,"%s %s %s",$1,$2,$3); + } + | /* empty */ { strcpy($$,""); } + ; + + +verb_phrase : VERB { strcpy($$,$1); strcat($$," "); } + | adverb VERB + { + sprintf($$,"%s %s",$1,$2); + } + ; + +adverb : ADVERB { strcpy($$,$1); } + | /* empty */ { strcpy($$,""); } + ; + +noun_phrase : DECLARATIVE NOUN + { + sprintf($$,"%s %s",$1,$2); + } + | CONDITIONAL declarative NOUN + { + sprintf($$,"%s %s %s",$1,$2,$3); + } + | NOUN { strcpy($$,$1); strcat($$," "); } + ; + +declarative : DECLARATIVE { strcpy($$,$1); } + | /* empty */ { strcpy($$,""); } + ; + +period : /* empty */ + | PERIOD + ; + + +%% + +/* Supplied main() and yyerror() functions. */ + +int main(int argc, char *argv[]) +{ + yyparse(); /* parse the file */ + return(0); +} + +int yyerror(char *message) +{ + extern FILE *yyout; + + fprintf(yyout,"\nError at line %5d. (%s) \n", + yylexlinenum,message); +} diff --git a/examples/manual/j2t.lex b/examples/manual/j2t.lex new file mode 100644 index 0000000..08fbd21 --- /dev/null +++ b/examples/manual/j2t.lex @@ -0,0 +1,442 @@ +/* + * j2t.lex : An example of the use (possibly abuse!) + * of start states. + */ + +%{ +#define MAX_STATES 1024 +#define TRUE 1 +#define FALSE 0 + +#define CHAPTER "@chapter" +#define SECTION "@section" +#define SSECTION "@subsection" +#define SSSECTION "@subsubsection" + +int states[MAX_STATES]; +int statep = 0; + +int need_closing = FALSE; + +char buffer[YY_BUF_SIZE]; + +extern char *yytext; + +/* + * set up the head of the *.texinfo file the program + * will produce. This is a standard texinfo header. + */ + +void print_header(void) +{ + printf("\\input texinfo @c -*-texinfo-*-\n"); + printf("@c %c**start of header\n",'%'); + printf("@setfilename jargon.info\n"); + printf("@settitle The New Hackers Dictionary\n"); + printf("@synindex fn cp\n"); + printf("@synindex vr cp\n"); + printf("@c %c**end of header\n",'%'); + printf("@setchapternewpage odd\n"); + printf("@finalout\n"); + printf("@c @smallbook\n"); + printf("\n"); + printf("@c ==========================================================\n\n"); + printf("@c This file was produced by j2t. Any mistakes are *not* the\n"); + printf("@c fault of the jargon file editors. \n"); + printf("@c ==========================================================\n\n"); + printf("@titlepage\n"); + printf("@title The New Hackers Dictionary\n"); + printf("@subtitle Version 2.9.10\n"); + printf("@subtitle Generated by j2t\n"); + printf("@author Eric S. Raymond, Guy L. Steel, Mark Crispin et al.\n"); + printf("@end titlepage\n"); + printf("@page\n"); + printf("\n@c ==========================================================\n"); + printf("\n\n"); + printf("@unnumbered Preface\n"); + printf("@c *******\n"); +} + +/* + * create the tail of the texinfo file produced. + */ + +void print_trailer(void) +{ + printf("\n@c ==========================================================\n"); + printf("@contents\n"); /* print the table of contents */ + printf("@bye\n\n"); +} + +/* + * write an underline under a section + * or chapter so we can find it later. + */ + +void write_underline(int len, int space, char ch) +{ + int loop; + + printf("@c "); + + for(loop=3; loop<space; loop++){ + printf(" "); + } + + while(len--){ + printf("%c",ch); + } + printf("\n\n"); +} + +/* + * check for texinfo special characters + * and escape them + */ + +char *check_and_convert(char *string) +{ + int buffpos = 0; + int len,loop; + + len = strlen(string); + for(loop=0; loop<len; loop++){ + if(string[loop] == '@' || string[loop] == '{' || string[loop] == '}'){ + buffer[buffpos++] = '@'; + buffer[buffpos++] = string[loop]; + } else { + buffer[buffpos++] = string[loop]; + } + } + buffer[buffpos] = '\0'; + return(buffer); +} + +/* + * write out a chapter,section, or subsection + * header + */ + +void write_block_header(char *type) +{ + int loop; + int len; + + (void)check_and_convert(yytext); + len = strlen(buffer); + for(loop=0; buffer[loop] != '\n';loop++) + ; + buffer[loop] = '\0'; + printf("%s %s\n",type,buffer); + write_underline(strlen(buffer),strlen(type)+1,'*'); +} + +%} + +/* + * the flex description starts here + */ + +%x HEADING EXAMPLE ENUM EXAMPLE2 +%x BITEM BITEM_ITEM +%s LITEM LITEM2 + +%% + +^#[^#]*"#" /* skip the header & trailer */ + /* chapters have asterisks under them + * and are terminated by a colon + */ +^[^\n:]+\n[*]+\n write_block_header(CHAPTER); + +^"= "[A-Z]" ="\n"="* { /* we create a seciton for each category */ + if(need_closing == TRUE){ + printf("@end table\n\n\n"); + } + need_closing = TRUE; + write_block_header(SECTION); + printf("\n\n@table @b\n"); + } + +"Examples:"[^\.]+ ECHO; + +"*"[^*\n]+"*" { /* @emph{}(emphasized) text */ + yytext[yyleng-1] = '\0'; + (void)check_and_convert(&yytext[1]); + printf("@i{%s}",buffer); + } + +"{{"[^}]+"}}" { /* special emphasis */ + yytext[yyleng-2] = '\0'; + (void)check_and_convert(&yytext[2]); + printf("@b{%s}",buffer); + } + +"{"[^}]+"}" { /* special emphasis */ + yytext[yyleng-1] = '\0'; + (void)check_and_convert(&yytext[1]); + printf("@b{%s}",buffer); + } + + /* escape some special texinfo characters */ +<INITIAL,LITEM,LITEM2,BITEM,ENUM,EXAMPLE,EXAMPLE2>"@" printf("@@"); +<INITIAL,LITEM,LITEM2,BITEM,ENUM,EXAMPLE,EXAMPLE2>"{" printf("@{"); +<INITIAL,LITEM,LITEM2,BITEM,ENUM,EXAMPLE,EXAMPLE2>"}" printf("@}"); + + /* + * reproduce @example code + */ + +":"\n+[^\n0-9*]+\n" "[^ ] { + int loop; + int len; + int cnt; + + printf(":\n\n@example \n"); + strcpy(buffer,yytext); + len = strlen(buffer); + cnt = 0; + for(loop=len; loop > 0;loop--){ + if(buffer[loop] == '\n') + cnt++; + if(cnt == 2) + break; + } + yyless(loop+1); + statep++; + states[statep] = EXAMPLE2; + BEGIN(EXAMPLE2); + } +<EXAMPLE,EXAMPLE2>^\n { + printf("@end example\n\n"); + statep--; + BEGIN(states[statep]); + } + + /* + * repoduce @enumerate lists + */ + +":"\n+[ \t]*[0-9]+"." { + int loop; + int len; + + printf(":\n\n@enumerate \n"); + strcpy(buffer,yytext); + len = strlen(buffer); + for(loop=len; loop > 0;loop--){ + if(buffer[loop] == '\n') + break; + } + yyless(loop); + statep++; + states[statep] = ENUM; + BEGIN(ENUM); + } + +<ENUM>"@" printf("@@"); +<ENUM>":"\n+" "[^0-9] { + printf(":\n\n@example\n"); + statep++; + states[statep] = EXAMPLE; + BEGIN(EXAMPLE); + } + + +<ENUM>\n[ \t]+[0-9]+"." { + printf("\n\n@item "); + } +<ENUM>^[^ ] | +<ENUM>\n\n\n[ \t]+[^0-9] { + printf("\n\n@end enumerate\n\n"); + statep--; + BEGIN(states[statep]); + } + + /* + * reproduce one kind of @itemize list + */ + +":"\n+":" { + int loop; + int len; + + printf(":\n\n@itemize @bullet \n"); + yyless(2); + statep++; + states[statep] = LITEM2; + BEGIN(LITEM2); + } +<LITEM2>^":".+":" { + (void)check_and_convert(&yytext[1]); + buffer[strlen(buffer)-1]='\0'; + printf("@item @b{%s:}\n",buffer); + } + +<LITEM2>\n\n\n+[^:\n] { + printf("\n\n@end itemize\n\n"); + ECHO; + statep--; + BEGIN(states[statep]); + } + + /* + * create a list out of the revision history part. + * We need the "Version" for this because it + * clashes with other rules otherwise. + */ + +:[\n]+"Version"[^:\n*]+":" { + int loop; + int len; + + printf(":\n\n@itemize @bullet \n"); + strcpy(buffer,yytext); + len = strlen(buffer); + for(loop=len; loop > 0;loop--){ + if(buffer[loop] == '\n') + break; + } + yyless(loop); + statep++; + states[statep] = LITEM; + BEGIN(LITEM); + } +<LITEM>^.+":" { + (void)check_and_convert(yytext); + buffer[strlen(buffer)-1]='\0'; + printf("@item @b{%s}\n\n",buffer); + } + +<LITEM>^[^:\n]+\n\n[^:\n]+\n { + int loop; + + strcpy(buffer,yytext); + for(loop=0; buffer[loop] != '\n'; loop++); + buffer[loop] = '\0'; + printf("%s\n",buffer); + printf("@end itemize\n\n"); + printf("%s",&buffer[loop+1]); + statep--; + BEGIN(states[statep]); + } + + /* + * reproduce @itemize @bullet lists + */ + +":"\n[ ]*"*" { + int loop; + int len; + + printf(":\n\n@itemize @bullet \n"); + len = strlen(buffer); + for(loop=0; loop < len;loop++){ + if(buffer[loop] == '\n') + break; + } + yyless((len-loop)+2); + statep++; + states[statep] = BITEM; + BEGIN(BITEM); + } + +<BITEM>^" "*"*" { + printf("@item"); + statep++; + states[statep] = BITEM_ITEM; + BEGIN(BITEM_ITEM); + } +<BITEM>"@" printf("@@"); +<BITEM>^\n { + printf("@end itemize\n\n"); + statep--; + BEGIN(states[statep]); + } +<BITEM_ITEM>[^\:]* { + printf(" @b{%s}\n\n",check_and_convert(yytext)); + } +<BITEM_ITEM>":" { + statep--; + BEGIN(states[statep]); + } + + /* + * recreate @chapter, @section etc. + */ + +^:[^:]* { + (void)check_and_convert(&yytext[1]); + statep++; + states[statep] = HEADING; + BEGIN(HEADING); + } +<HEADING>:[^\n] { + printf("@item @b{%s}\n",buffer); + write_underline(strlen(buffer),6,'~'); + statep--; + BEGIN(states[statep]); + } +<HEADING>:\n"*"* { + if(need_closing == TRUE){ + printf("@end table\n\n\n"); + need_closing = FALSE; + } + printf("@chapter %s\n",buffer); + write_underline(strlen(buffer),9,'*'); + statep--; + BEGIN(states[statep]); + } +<HEADING>:\n"="* { + if(need_closing == TRUE){ + printf("@end table\n\n\n"); + need_closing = FALSE; + } + printf("@section %s\n",buffer); + write_underline(strlen(buffer),9,'='); + statep--; + BEGIN(states[statep]); + } +<HEADING>"@" printf("@@"); +<HEADING>:\n"-"* { + if(need_closing == TRUE){ + printf("@end table\n\n\n"); + need_closing = FALSE; + } + printf("@subsection %s\n",buffer); + write_underline(strlen(buffer),12,'-'); + statep--; + BEGIN(states[statep]); + } + + /* + * recreate @example text + */ + +^" " { + printf("@example\n"); + statep++; + states[statep] = EXAMPLE; + BEGIN(EXAMPLE); + } +<EXAMPLE>^" " +. ECHO; + +%% + +/* + * initialise and go. + */ + +int main(int argc, char *argv[]) +{ + states[0] = INITIAL; + statep = 0; + print_header(); + yylex(); + print_trailer(); + return(0); +} + + + diff --git a/examples/manual/myname.lex b/examples/manual/myname.lex new file mode 100644 index 0000000..2e36095 --- /dev/null +++ b/examples/manual/myname.lex @@ -0,0 +1,15 @@ +/* + * + * myname.lex : A sample Flex program + * that does token replacement. + */ + +%% + +%NAME { printf("%s",getenv("LOGNAME")); } +%HOST { printf("%s",getenv("HOST")); } +%HOSTTYPE { printf("%s",getenv("HOSTTYPE"));} +%HOME { printf("%s",getenv("HOME")); } + +%% + diff --git a/examples/manual/myname.txt b/examples/manual/myname.txt new file mode 100644 index 0000000..0631840 --- /dev/null +++ b/examples/manual/myname.txt @@ -0,0 +1,6 @@ +Hello, my name name is %NAME. Actually +"%NAME" isn't my real name, it is the +alias I use when I'm on %HOST, which +is the %HOSTTYPE I use. My HOME +directory is %HOME. + diff --git a/examples/manual/myname2.lex b/examples/manual/myname2.lex new file mode 100644 index 0000000..cef55d8 --- /dev/null +++ b/examples/manual/myname2.lex @@ -0,0 +1,19 @@ +/* + * myname2.lex : A sample Flex program + * that does token replacement. + */ + +%{ +#include <stdio.h> +%} + +%x STRING +%% +\" ECHO; BEGIN(STRING); +<STRING>[^\"\n]* ECHO; +<STRING>\" ECHO; BEGIN(INITIAL); + +%NAME { printf("%s",getenv("LOGNAME")); } +%HOST { printf("%s",getenv("HOST")); } +%HOSTTYPE { printf("%s",getenv("HOSTTYPE"));} +%HOME { printf("%s",getenv("HOME")); } diff --git a/examples/manual/numbers.lex b/examples/manual/numbers.lex new file mode 100644 index 0000000..6484d6e --- /dev/null +++ b/examples/manual/numbers.lex @@ -0,0 +1,145 @@ +/* + * numbers.lex : An example of the definitions and techniques + * for scanning numbers + */ + +%{ +#include <stdio.h> + +#define UNSIGNED_LONG_SYM 1 +#define SIGNED_LONG_SYM 2 +#define UNSIGNED_SYM 3 +#define SIGNED_SYM 4 +#define LONG_DOUBLE_SYM 5 +#define FLOAT_SYM 6 + +union _yylval { + long double ylong_double; + float yfloat; + unsigned long yunsigned_long; + unsigned yunsigned; + long ysigned_long; + int ysigned; +} yylval; + +%} + +digit [0-9] +hex_digit [0-9a-fA-F] +oct_digit [0-7] + +exponent [eE][+-]?{digit}+ +i {digit}+ +float_constant ({i}\.{i}?|{i}?\.{i}){exponent}? +hex_constant 0[xX]{hex_digit}+ +oct_constant 0{oct_digit}* +int_constant {digit}+ +long_ext [lL] +unsigned_ext [uU] +float_ext [fF] +ulong_ext {long_ext}{unsigned_ext}|{unsigned_ext}{long_ext} + +%% + +{hex_constant}{ulong_ext} { /* we need to skip the "0x" part */ + sscanf(&yytext[2],"%lx",&yylval.yunsigned_long); + return(UNSIGNED_LONG_SYM); + } +{hex_constant}{long_ext} { + sscanf(&yytext[2],"%lx",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{hex_constant}{unsigned_ext} { + sscanf(&yytext[2],"%x",&yylval.yunsigned); + return(UNSIGNED_SYM); + } +{hex_constant} { /* use %lx to protect against overflow */ + sscanf(&yytext[2],"%lx",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{oct_constant}{ulong_ext} { + sscanf(yytext,"%lo",&yylval.yunsigned_long); + return(UNSIGNED_LONG_SYM); + } +{oct_constant}{long_ext} { + sscanf(yytext,"%lo",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{oct_constant}{unsigned_ext} { + sscanf(yytext,"%o",&yylval.yunsigned); + return(UNSIGNED_SYM); + } +{oct_constant} { /* use %lo to protect against overflow */ + sscanf(yytext,"%lo",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{int_constant}{ulong_ext} { + sscanf(yytext,"%ld",&yylval.yunsigned_long); + return(UNSIGNED_LONG_SYM); + } +{int_constant}{long_ext} { + sscanf(yytext,"%ld",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{int_constant}{unsigned_ext} { + sscanf(yytext,"%d",&yylval.yunsigned); + return(UNSIGNED_SYM); + } +{int_constant} { /* use %ld to protect against overflow */ + sscanf(yytext,"%ld",&yylval.ysigned_long); + return(SIGNED_LONG_SYM); + } +{float_constant}{long_ext} { + sscanf(yytext,"%lf",&yylval.ylong_double); + return(LONG_DOUBLE_SYM); + } +{float_constant}{float_ext} { + sscanf(yytext,"%f",&yylval.yfloat); + return(FLOAT_SYM); + } +{float_constant} { /* use %lf to protect against overflow */ + sscanf(yytext,"%lf",&yylval.ylong_double); + return(LONG_DOUBLE_SYM); + } +%% + +int main(void) +{ + int code; + + while((code = yylex())){ + printf("yytext : %s\n",yytext); + switch(code){ + case UNSIGNED_LONG_SYM: + printf("Type of number : UNSIGNED LONG\n"); + printf("Value of number : %lu\n",yylval.yunsigned_long); + break; + case SIGNED_LONG_SYM: + printf("Type of number : SIGNED LONG\n"); + printf("Value of number : %ld\n",yylval.ysigned_long); + break; + case UNSIGNED_SYM: + printf("Type of number : UNSIGNED\n"); + printf("Value of number : %u\n",yylval.yunsigned); + break; + case SIGNED_SYM: + printf("Type of number : SIGNED\n"); + printf("Value of number : %d\n",yylval.ysigned); + break; + case LONG_DOUBLE_SYM: + printf("Type of number : LONG DOUBLE\n"); + printf("Value of number : %lf\n",yylval.ylong_double); + break; + case FLOAT_SYM: + printf("Type of number : FLOAT\n"); + printf("Value of number : %f\n",yylval.yfloat); + break; + default: + printf("Type of number : UNDEFINED\n"); + printf("Value of number : UNDEFINED\n"); + break; + } + } + return(0); +} + diff --git a/examples/manual/pas_include.lex b/examples/manual/pas_include.lex new file mode 100644 index 0000000..58cf590 --- /dev/null +++ b/examples/manual/pas_include.lex @@ -0,0 +1,78 @@ +/* + * eof_rules.lex : An example of using multiple buffers + * EOF rules, and start states + */ + +%{ + +#define MAX_NEST 10 + +YY_BUFFER_STATE include_stack[MAX_NEST]; +int include_count = -1; + +%} + + +%x INCLUDE +%x COMMENT + + +%% + +"{" BEGIN(COMMENT); + +<COMMENT>"}" BEGIN(INITIAL); +<COMMENT>"$include"[ \t]*"(" BEGIN(INCLUDE); +<COMMENT>[ \t]* /* skip whitespace */ + +<INCLUDE>")" BEGIN(COMMENT); +<INCLUDE>[ \t]* /* skip whitespace */ +<INCLUDE>[^ \t\n() ]+ { /* get the include file name */ + if ( include_count >= MAX_NEST){ + fprintf( stderr, "Too many include files" ); + exit( 1 ); + } + + include_stack[++include_count] = YY_CURRENT_BUFFER; + + yyin = fopen( yytext, "r" ); + if ( ! yyin ){ + fprintf( stderr, "Unable to open %s",yytext); + exit( 1 ); + } + + yy_switch_to_buffer(yy_create_buffer(yyin,YY_BUF_SIZE)); + + BEGIN(INITIAL); + } +<INCLUDE><<EOF>> + { + fprintf( stderr, "EOF in include" ); + yyterminate(); + } +<COMMENT><<EOF>> + { + fprintf( stderr, "EOF in comment" ); + yyterminate(); + } +<<EOF>> { + if ( include_count <= 0 ){ + yyterminate(); + } else { + yy_delete_buffer(include_stack[include_count--] ); + yy_switch_to_buffer(include_stack[include_count] ); + BEGIN(INCLUDE); + } + } +[a-z]+ ECHO; +.|\n ECHO; + + + + + + + + + + diff --git a/examples/manual/pascal.lex b/examples/manual/pascal.lex new file mode 100644 index 0000000..d406bbe --- /dev/null +++ b/examples/manual/pascal.lex @@ -0,0 +1,120 @@ +/* + * pascal.lex: An example PASCAL scanner + * + */ + +%{ +#include <stdio.h> +#include "y.tab.h" + +int line_number = 0; + +void yyerror(char *message); + +%} + +%x COMMENT1 COMMENT2 + +white_space [ \t]* +digit [0-9] +alpha [A-Za-z_] +alpha_num ({alpha}|{digit}) +hex_digit [0-9A-F] +identifier {alpha}{alpha_num}* +unsigned_integer {digit}+ +hex_integer ${hex_digit}{hex_digit}* +exponent e[+-]?{digit}+ +i {unsigned_integer} +real ({i}\.{i}?|{i}?\.{i}){exponent}? +string \'([^'\n]|\'\')+\' +bad_string \'([^'\n]|\'\')+ + +%% + +"{" BEGIN(COMMENT1); +<COMMENT1>[^}\n]+ +<COMMENT1>\n ++line_number; +<COMMENT1><<EOF>> yyerror("EOF in comment"); +<COMMENT1>"}" BEGIN(INITIAL); + +"(*" BEGIN(COMMENT2); +<COMMENT2>[^)*\n]+ +<COMMENT2>\n ++line_number; +<COMMENT2><<EOF>> yyerror("EOF in comment"); +<COMMENT2>"*)" BEGIN(INITIAL); +<COMMENT2>[*)] + + /* note that FILE and BEGIN are already + * defined in FLEX or C so they can't + * be used. This can be overcome in + * a cleaner way by defining all the + * tokens to start with TOK_ or some + * other prefix. + */ + +and return(AND); +array return(ARRAY); +begin return(_BEGIN); +case return(CASE); +const return(CONST); +div return(DIV); +do return(DO); +downto return(DOWNTO); +else return(ELSE); +end return(END); +file return(_FILE); +for return(FOR); +function return(FUNCTION); +goto return(GOTO); +if return(IF); +in return(IN); +label return(LABEL); +mod return(MOD); +nil return(NIL); +not return(NOT); +of return(OF); +packed return(PACKED); +procedure return(PROCEDURE); +program return(PROGRAM); +record return(RECORD); +repeat return(REPEAT); +set return(SET); +then return(THEN); +to return(TO); +type return(TYPE); +until return(UNTIL); +var return(VAR); +while return(WHILE); +with return(WITH); + +"<="|"=<" return(LEQ); +"=>"|">=" return(GEQ); +"<>" return(NEQ); +"=" return(EQ); + +".." return(DOUBLEDOT); + +{unsigned_integer} return(UNSIGNED_INTEGER); +{real} return(REAL); +{hex_integer} return(HEX_INTEGER); +{string} return{STRING}; +{bad_string} yyerror("Unterminated string"); + +{identifier} return(IDENTIFIER); + +[*/+\-,^.;:()\[\]] return(yytext[0]); + +{white_space} /* do nothing */ +\n line_number += 1; +. yyerror("Illegal input"); + +%% + +void yyerror(char *message) +{ + fprintf(stderr,"Error: \"%s\" in line %d. Token = %s\n", + message,line_number,yytext); + exit(1); +} + + diff --git a/examples/manual/reject.lex b/examples/manual/reject.lex new file mode 100644 index 0000000..a7b817f --- /dev/null +++ b/examples/manual/reject.lex @@ -0,0 +1,12 @@ +/* + * reject.lex: An example of REJECT and unput() + * misuse. + */ + +%% +UNIX { + unput('U'); unput('N'); unput('G'); unput('\0'); + REJECT; + } +GNU printf("GNU is Not Unix!\n"); +%% diff --git a/examples/manual/replace.lex b/examples/manual/replace.lex new file mode 100644 index 0000000..c5c8d87 --- /dev/null +++ b/examples/manual/replace.lex @@ -0,0 +1,33 @@ +/* + * replace.lex : A simple filter for renaming + * parts of flex of bison generated + * scanners or parsers. + */ + +%{ +#include <stdio.h> + +char lower_replace[1024]; +char upper_replace[1024]; + +%} + +%% + +"yy" printf("%s",lower_replace); +"YY" printf("%s",upper_replace); +, ECHO; + +%% + +int main(int argc, char *argv[]) +{ + if(argc < 2){ + printf("Usage %s lower UPPER\n",argv[0]); + exit(1); + } + strcpy(lower_replace,argv[1]); + strcpy(upper_replace,argv[2]); + yylex(); + return(0); +} diff --git a/examples/manual/string1.lex b/examples/manual/string1.lex new file mode 100644 index 0000000..b62ed88 --- /dev/null +++ b/examples/manual/string1.lex @@ -0,0 +1,98 @@ +/* + * string1.lex: Handling strings by using input() + */ + +%{ +#include <ctype.h> +#include <malloc.h> + +#define ALLOC_SIZE 32 /* for (re)allocating the buffer */ + +#define isodigit(x) ((x) >= '0' && (x) <= '7') +#define hextoint(x) (isdigit((x)) ? (x) - '0' : ((x) - 'A') + 10) + +void yyerror(char *message) +{ + printf("\nError: %s\n",message); +} + +%} + +%% + +\" { + int inch,count,max_size; + char *buffer; + int temp; + + buffer = malloc(ALLOC_SIZE); + max_size = ALLOC_SIZE; + inch = input(); + count = 0; + while(inch != EOF && inch != '"' && inch != '\n'){ + if(inch == '\\'){ + inch = input(); + switch(inch){ + case '\n': inch = input(); break; + case 'b' : inch = '\b'; break; + case 't' : inch = '\t'; break; + case 'n' : inch = '\n'; break; + case 'v' : inch = '\v'; break; + case 'f' : inch = '\f'; break; + case 'r' : inch = '\r'; break; + case 'X' : + case 'x' : inch = input(); + if(isxdigit(inch)){ + temp = hextoint(toupper(inch)); + inch = input(); + if(isxdigit(inch)){ + temp = (temp << 4) + hextoint(toupper(inch)); + } else { + unput(inch); + } + inch = temp; + } else { + unput(inch); + inch = 'x'; + } + break; + default: + if(isodigit(inch)){ + temp = inch - '0'; + inch = input(); + if(isodigit(inch)){ + temp = (temp << 3) + (inch - '0'); + } else { + unput(inch); + goto done; + } + inch = input(); + if(isodigit(inch)){ + temp = (temp << 3) + (inch - '0'); + } else { + unput(inch); + } + done: + inch = temp; + } + } + } + buffer[count++] = inch; + if(count >= max_size){ + buffer = realloc(buffer,max_size + ALLOC_SIZE); + max_size += ALLOC_SIZE; + } + inch = input(); + } + if(inch == EOF || inch == '\n'){ + yyerror("Unterminated string."); + } + buffer[count] = '\0'; + printf("String = \"%s\"\n",buffer); + free(buffer); + } +. +\n +%% + + diff --git a/examples/manual/string2.lex b/examples/manual/string2.lex new file mode 100644 index 0000000..2c9d35f --- /dev/null +++ b/examples/manual/string2.lex @@ -0,0 +1,94 @@ +/* + * string2.lex: An example of using scanning strings + * by using start states. + */ + +%{ +#include <ctype.h> +#include <malloc.h> + +#define isodigit(x) ((x) >= '0' && (x) <= '7') +#define hextoint(x) (isdigit((x)) ? (x) - '0' : ((x) - 'A') + 10) + +char *buffer = NULL; +int buffer_size = 0; + +void yyerror(char *message) +{ + printf("\nError: %s\n",message); +} + +%} + +%x STRING + +hex (x|X)[0-9a-fA-F]{1,2} +oct [0-7]{1,3} + +%% + +\" { + buffer = malloc(1); + buffer_size = 1; strcpy(buffer,""); + BEGIN(STRING); + } +<STRING>\n { + yyerror("Unterminated string"); + free(buffer); + BEGIN(INITIAL); + } +<STRING><<EOF>> { + yyerror("EOF in string"); + free(buffer); + BEGIN(INITIAL); + } +<STRING>[^\\\n"] { + buffer = realloc(buffer,buffer_size+yyleng+1); + buffer_size += yyleng; + strcat(buffer,yytext); + } +<STRING>\\\n /* ignore this */ +<STRING>\\{hex} { + int temp =0,loop = 0; + for(loop=yyleng-2; loop>0; loop--){ + temp <<= 4; + temp += hextoint(toupper(yytext[yyleng-loop])); + } + buffer = realloc(buffer,buffer_size+1); + buffer[buffer_size-1] = temp; + buffer[buffer_size] = '\0'; + buffer_size += 1; + } +<STRING>\\{oct} { + int temp =0,loop = 0; + for(loop=yyleng-1; loop>0; loop--){ + temp <<= 3; + temp += (yytext[yyleng-loop] - '0'); + } + buffer = realloc(buffer,buffer_size+1); + buffer[buffer_size-1] = temp; + buffer[buffer_size] = '\0'; + buffer_size += 1; + } +<STRING>\\[^\n] { + buffer = realloc(buffer,buffer_size+1); + switch(yytext[yyleng-1]){ + case 'b' : buffer[buffer_size-1] = '\b'; break; + case 't' : buffer[buffer_size-1] = '\t'; break; + case 'n' : buffer[buffer_size-1] = '\n'; break; + case 'v' : buffer[buffer_size-1] = '\v'; break; + case 'f' : buffer[buffer_size-1] = '\f'; break; + case 'r' : buffer[buffer_size-1] = '\r'; break; + default : buffer[buffer_size-1] = yytext[yyleng-1]; + } + buffer[buffer_size] = '\0'; + buffer_size += 1; + } +<STRING>\" { + printf("string = \"%s\"",buffer); + free(buffer); + BEGIN(INITIAL); + } +%% + + diff --git a/examples/manual/strtest.dat b/examples/manual/strtest.dat new file mode 100644 index 0000000..28a0681 --- /dev/null +++ b/examples/manual/strtest.dat @@ -0,0 +1,21 @@ +"This is a string" +"The next string will be empty" +"" +"This is a string with a \b(\\b) in it" +"This is a string with a \t(\\t) in it" +"This is a string with a \n(\\n) in it" +"This is a string with a \v(\\v) in it" +"This is a string with a \f(\\f) in it" +"This is a string with a \r(\\r) in it" +"This is a string with a \"(\\\") in it" +"This is a string with a \z(\\z) in it" +"This is a string with a \X4a(\\X4a) in it" +"This is a string with a \x4a(\\x4a) in it" +"This is a string with a \x7(\\x7) in it" +"This is a string with a \112(\\112) in it" +"This is a string with a \043(\\043) in it" +"This is a string with a \7(\\7) in it" +"This is a multi-line \ +string" +"This is an unterminated string +"This is an unterminated string too diff --git a/examples/manual/unput.lex b/examples/manual/unput.lex new file mode 100644 index 0000000..161471a --- /dev/null +++ b/examples/manual/unput.lex @@ -0,0 +1,32 @@ +/* + * unput.l : An example of what *not* + * to do with unput(). + */ + + +%{ +#include <stdio.h> + +void putback_yytext(void); +%} + +%% +foobar putback_yytext(); +raboof putback_yytext(); +%% + +void putback_yytext(void) +{ + int i; + int l = strlen(yytext); + char buffer[YY_BUF_SIZE]; + + strcpy(buffer,yytext); + printf("Got: %s\n",yytext); + for(i=0; i<l; i++){ + unput(buffer[i]); + } +} + + + diff --git a/examples/manual/user_act.lex b/examples/manual/user_act.lex new file mode 100644 index 0000000..156d5f9 --- /dev/null +++ b/examples/manual/user_act.lex @@ -0,0 +1,31 @@ +%{ + +#include <ctype.h> + +void user_action(void); + +#define YY_USER_ACTION user_action(); + +%} + +%% + +.* ECHO; +\n ECHO; + +%% + +void user_action(void) +{ + int loop; + + for(loop=0; loop<yyleng; loop++){ + if(islower(yytext[loop])){ + yytext[loop] = toupper(yytext[loop]); + } + } +} + + + + diff --git a/examples/manual/userinit.lex b/examples/manual/userinit.lex new file mode 100644 index 0000000..9a56471 --- /dev/null +++ b/examples/manual/userinit.lex @@ -0,0 +1,30 @@ +%{ +#define YY_USER_INIT open_input_file() + +extern FILE *yyin; + +void open_input_file(void) +{ + char *file_name,buffer[1024]; + + yyin = NULL; + + while(yyin == NULL){ + printf("Input file: "); + file_name = fgets(buffer,1024,stdin); + if(file_name){ + file_name[strlen(file_name)-1] = '\0'; + yyin = fopen(file_name,"r"); + if(yyin == NULL){ + printf("Unable to open \"%s\"\n",file_name); + } + } else { + printf("stdin\n"); + yyin = stdin; + break; + } + } +} + +%} +%% diff --git a/examples/manual/wc.lex b/examples/manual/wc.lex new file mode 100644 index 0000000..7cae6a1 --- /dev/null +++ b/examples/manual/wc.lex @@ -0,0 +1,122 @@ +%{ + +/* + * wc.lex : A simple example of using FLEX + * to create a wc-like utility. + * + * See MISC/fastwc/ in the flex distribution for examples + * of how to write this scanner for maximum performance. + */ + +int numchars = 0; +int numwords = 0; +int numlines = 0; +int totchars = 0; +int totwords = 0; +int totlines = 0; + +/* + * rules start from here + */ + +%} + +%% + +[\n] { numchars++; numlines++; } +[\r] { numchars++; } +[^ \t\n]+ { numwords++; numchars += yyleng; } +. { numchars++; } + +%% + +/* + * additional C code start from here. This supplies + * all the argument processing etc. + */ + +int main(int argc, char *argv[]) +{ + int loop,first=1; + int lflag = 0; /* 1 if we count # of lines */ + int wflag = 0; /* 1 if we count # of words */ + int cflag = 0; /* 1 if we count # of characters */ + int fflag = 0; /* 1 if we have a file name */ + + for(loop=1; loop<argc; loop++){ + if(argv[loop][0] == '-'){ + switch(argv[loop][1]){ + case 'l': + lflag = 1; + break; + case 'w': + wflag = 1; + break; + case 'c': + cflag = 1; + break; + default: + fprintf(stderr,"unknown option -%c\n", + argv[loop][1]); + } + } + } + if(lflag == 0 && wflag == 0 && cflag == 0){ + lflag = wflag = cflag = 1; /* default to all on */ + } + + for(loop=1; loop<argc; loop++){ + if(argv[loop][0] != '-'){ + fflag = 1; + numlines = numchars = numwords = 0; + if((yyin = fopen(argv[loop],"rb")) != NULL){ + if(first){ + first = 0; + } else { + YY_NEW_FILE; + } + (void) yylex(); + fclose(yyin); + totwords += numwords; + totchars += numchars; + totlines += numlines; + printf("file : %25s :",argv[loop]) ; + if(lflag){ + fprintf(stdout,"lines %5d ",numlines); + } + if(cflag){ + fprintf(stdout,"characters %5d ",numchars); + } + if(wflag){ + fprintf(stdout,"words %5d ",numwords); + } + fprintf(stdout,"\n"); + }else{ + fprintf(stderr,"wc : file not found %s\n",argv[loop]); + } + } + } + if(!fflag){ + fprintf(stderr,"usage : wc [-l -w -c] file [file...]\n"); + fprintf(stderr,"-l = count lines\n"); + fprintf(stderr,"-c = count characters\n"); + fprintf(stderr,"-w = count words\n"); + exit(1); + } + for(loop=0;loop<79; loop++){ + fprintf(stdout,"-"); + } + fprintf(stdout,"\n"); + fprintf(stdout,"total : %25s ","") ; + if(lflag){ + fprintf(stdout,"lines %5d ",totlines); + } + if(cflag){ + fprintf(stdout,"characters %5d ",totchars); + } + if(wflag){ + fprintf(stdout,"words %5d ",totwords); + } + fprintf(stdout,"\n"); + return(0); +} diff --git a/examples/manual/yymore.lex b/examples/manual/yymore.lex new file mode 100644 index 0000000..27d7012 --- /dev/null +++ b/examples/manual/yymore.lex @@ -0,0 +1,29 @@ +/* + * yymore.lex: An example of using yymore() + * to good effect. + */ + +%{ +#include <memory.h> + +void yyerror(char *message) +{ + printf("Error: %s\n",message); +} + +%} + +%x STRING + +%% +\" BEGIN(STRING); + +<STRING>[^\\\n"]* yymore(); +<STRING><<EOF>> yyerror("EOF in string."); BEGIN(INITIAL); +<STRING>\n yyerror("Unterminated string."); BEGIN(INITIAL); +<STRING>\\\n yymore(); +<STRING>\" { + yytext[yyleng-1] = '\0'; + printf("string = \"%s\"",yytext); BEGIN(INITIAL); + } +%% diff --git a/examples/manual/yymore2.lex b/examples/manual/yymore2.lex new file mode 100644 index 0000000..f49ea23 --- /dev/null +++ b/examples/manual/yymore2.lex @@ -0,0 +1,33 @@ +/* + * yymore.lex: An example of using yymore() + * to good effect. + */ + +%{ +#include <memory.h> + +void yyerror(char *message) +{ + printf("Error: %s\n",message); +} + +%} + +%x STRING + +%% +\" BEGIN(STRING); + +<STRING>[^\\\n"]* yymore(); +<STRING><<EOF>> yyerror("EOF in string."); BEGIN(INITIAL); +<STRING>\n yyerror("Unterminated string."); BEGIN(INITIAL); +<STRING>\\\n { + bcopy(yytext,yytext+2,yyleng-2); + yytext += 2; yyleng -= 2; + yymore(); + } +<STRING>\" { + yyleng -= 1; yytext[yyleng] = '\0'; + printf("string = \"%s\"",yytext); BEGIN(INITIAL); + } +%% diff --git a/examples/manual/yymoretest.dat b/examples/manual/yymoretest.dat new file mode 100644 index 0000000..614c3c4 --- /dev/null +++ b/examples/manual/yymoretest.dat @@ -0,0 +1,7 @@ +"This is a test \ +of multi-line string \ +scanning in flex. \ +This may be breaking some law \ +of usage though..." + + diff --git a/examples/testxxLexer.l b/examples/testxxLexer.l new file mode 100644 index 0000000..9421541 --- /dev/null +++ b/examples/testxxLexer.l @@ -0,0 +1,58 @@ + // An example of using the flex C++ scanner class. + +%option C++ noyywrap + +%{ +int mylineno = 0; +%} + +string \"[^\n"]+\" + +ws [ \t]+ + +alpha [A-Za-z] +dig [0-9] +name ({alpha}|{dig}|\$)({alpha}|{dig}|\_|\.|\-|\/|\$)* +num1 [-+]?{dig}+\.?([eE][-+]?{dig}+)? +num2 [-+]?{dig}*\.{dig}+([eE][-+]?{dig}+)? +number {num1}|{num2} + +%% + +{ws} /* skip blanks and tabs */ + +"/*" { + int c; + + while((c = yyinput()) != 0) + { + if(c == '\n') + ++mylineno; + + else if(c == '*') + { + if((c = yyinput()) == '/') + break; + else + unput(c); + } + } + } + +{number} cout << "number " << YYText() << '\n'; + +\n mylineno++; + +{name} cout << "name " << YYText() << '\n'; + +{string} cout << "string " << YYText() << '\n'; + +%% + +int main( int /* argc */, char** /* argv */ ) + { + FlexLexer* lexer = new yyFlexLexer; + while(lexer->yylex() != 0) + ; + return 0; + } |