Fix M4 quoting of section 3.

This fixes M4 quoting of section 3 of the input file, including escape
sequences and character constants.

Tests were added to verify the behavior in section 3 with respect to
quoting. Both escaping of quotes and quoting of potential macro-start
characters are tested. Existing tests were also fixed to account for the new -- and now correct -- behavior. Many tests relied on the old behavior of expanding M4 macros
in section 3.  They needed to be updated for the new behavior.

28 files changed, 177 insertions, 145 deletions

diff --git a/src/filter.c b/src/filter.c
index 1ac199f..9d0ace5 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -390,7 +390,7 @@ int filter_fix_linedirs (struct filter *chain)
 				/* Adjust the line directives. */
 				in_gen = true;
 				snprintf (buf, readsz, "#line %d \"%s\"\n",
-					  lineno + 1, filename);
+					  lineno, filename);
 			}
 			else {
 				/* it's a #line directive for code we didn't write */
diff --git a/src/flexdef.h b/src/flexdef.h
index a727ede..a4d1896 100644
--- a/src/flexdef.h
+++ b/src/flexdef.h
@@ -1063,9 +1063,10 @@ extern struct Buf defs_buf;    /* a char* buffer to save #define'd some symbols
 extern struct Buf yydmap_buf;  /* a string buffer to hold yydmap elements */
 extern struct Buf m4defs_buf;  /* Holds m4 definitions. */
 extern struct Buf top_buf;     /* contains %top code. String buffer. */
+extern bool no_section3_escape; /* True if the undocumented option --unsafe-no-m4-sect3-escape was passed */
 
 /* For blocking out code from the header file. */
-#define OUT_BEGIN_CODE() outn("m4_ifdef( [[M4_YY_IN_HEADER]],,[[")
+#define OUT_BEGIN_CODE() outn("m4_ifdef( [[M4_YY_IN_HEADER]],,[[m4_dnl")
 #define OUT_END_CODE()   outn("]])")
 
 /* For setjmp/longjmp (instead of calling exit(2)). Linkage in main.c */
diff --git a/src/main.c b/src/main.c
index 805c48b..e0ffbc3 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1427,6 +1427,10 @@ void flexinit (int argc, char **argv)
 			break;
 		case OPT_HEX:
 			trace_hex = 1;
+                        break;
+                case OPT_NO_SECT3_ESCAPE:
+                        no_section3_escape = true;
+                        break;
 		}		/* switch */
 	}			/* while scanopt() */
 
diff --git a/src/misc.c b/src/misc.c
index 753ada3..e4e3f88 100644
--- a/src/misc.c
+++ b/src/misc.c
@@ -362,8 +362,8 @@ void line_directive_out (FILE *output_file, int do_infile)
 	s3 = &filename[sizeof (filename) - 2];
 
 	while (s2 < s3 && *s1) {
-		if (*s1 == '\\')
-			/* Escape the '\' */
+		if (*s1 == '\\' || *s1 == '"')
+			/* Escape the '\' or '"' */
 			*s2++ = '\\';
 
 		*s2++ = *s1++;
@@ -512,7 +512,8 @@ unsigned char myesc (unsigned char array[])
 		{		/* \<octal> */
 			int     sptr = 1;
 
-			while (isascii (array[sptr]) &&
+			while (sptr <= 3 &&
+                               isascii (array[sptr]) &&
 			       isdigit (array[sptr]))
 				/* Don't increment inside loop control
 				 * because if isdigit() is a macro it might
diff --git a/src/options.c b/src/options.c
index 3f3a39f..366bc2e 100644
--- a/src/options.c
+++ b/src/options.c
@@ -271,7 +271,8 @@ optspec_t flexopts[] = {
 	,
 	{"--noyyset_lloc", OPT_NO_YYSET_LLOC, 0}
 	,
-
+        {"--unsafe-no-m4-sect3-escape", OPT_NO_SECT3_ESCAPE, 0}
+        ,
 	{0, 0, 0}		/* required final NULL entry. */
 };
 
diff --git a/src/options.h b/src/options.h
index 8555343..5b51c23 100644
--- a/src/options.h
+++ b/src/options.h
@@ -125,7 +125,8 @@ enum flexopt_flag_t {
 	OPT_YYCLASS,
 	OPT_YYLINENO,
 	OPT_YYMORE,
-	OPT_YYWRAP
+	OPT_YYWRAP,
+        OPT_NO_SECT3_ESCAPE,
 };
 
 #endif
diff --git a/src/scan.l b/src/scan.l
index afafa7d..9a2bd83 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -41,6 +41,9 @@ extern const char *escaped_qstart, *escaped_qend;
 #define M4QSTART "[["
 #define M4QEND "]]"
 
+#define SECT3_ESCAPED_QSTART "[" M4QEND M4QSTART "[" M4QEND M4QSTART
+#define SECT3_ESCAPED_QEND M4QEND "]" M4QSTART M4QEND "]" M4QSTART
+
 #define ACTION_ECHO add_action( yytext )
 #define ACTION_IFDEF(def, should_define) \
 	{ \
@@ -110,6 +113,8 @@ extern const char *escaped_qstart, *escaped_qend;
 %x GROUP_MINUS_PARAMS
 %x EXTENDED_COMMENT
 %x COMMENT_DISCARD
+%x SECT3_NOESCAPE
+%x CHARACTER_CONSTANT
 
 WS		[[:blank:]]+
 OPTWS		[[:blank:]]*
@@ -240,8 +245,8 @@ M4QEND      "]]"
 			}
 	.		/* ignore spurious characters */
 }
-<ACTION,CODEBLOCK,ACTION_STRING,PERCENT_BRACE_ACTION,COMMENT>{
-   "M4"|"YY"|"m4" add_action(M4QSTART); ACTION_ECHO; add_action(M4QEND);
+<ACTION,CODEBLOCK,ACTION_STRING,PERCENT_BRACE_ACTION,COMMENT,CHARACTER_CONSTANT>{
+   M4|YY|m4 add_action(M4QSTART); ACTION_ECHO; add_action(M4QEND);
    {M4QSTART}   ACTION_ECHO_QSTART;
    {M4QEND}     ACTION_ECHO_QEND;
 }
@@ -285,9 +290,9 @@ M4QEND      "]]"
        buf_strnappend(&top_buf, M4QEND, 2);
     }
 
-    [^{}\r\n]  {
-                buf_strnappend(&top_buf, yytext, yyleng);
-               }
+    ([^{}\r\nmMY\[\]]+)|[^{}\r\n]  {
+       buf_strnappend(&top_buf, yytext, yyleng);
+    }
 
     <<EOF>>     {
                 linenum = brace_start_line;
@@ -920,10 +925,11 @@ nmstr[yyleng - 2 - end_is_ws] = '\0';  /* chop trailing brace */
 <ACTION>{
 	"{"		ACTION_ECHO; ++bracelevel;
 	"}"		ACTION_ECHO; --bracelevel;
-	[^[:alpha:]_{}"'/\n\[\]]+	ACTION_ECHO;
+	[^[:alpha:]_{}\"'/\n\[\]]+	ACTION_ECHO;
     [\[\]]      ACTION_ECHO;
 	{NAME}		ACTION_ECHO;
-	"'"([^'\\\n]|\\.)*"'"	ACTION_ECHO; /* character constant */
+        "'"([^\'\\\n]|\\.)"'" ACTION_ECHO; /* character constant */
+        "'"             ACTION_ECHO; yy_push_state(CHARACTER_CONSTANT);
 	\"		ACTION_ECHO; BEGIN(ACTION_STRING);
 	{NL}		{
 			++linenum;
@@ -941,14 +947,20 @@ nmstr[yyleng - 2 - end_is_ws] = '\0';  /* chop trailing brace */
 }
 
 <ACTION_STRING>{
-	[^]"\\\n[]+	ACTION_ECHO;
+	[^\]\"\\\n\[MmY]+	ACTION_ECHO;
+	\"		ACTION_ECHO; BEGIN(ACTION);
+}
+<CHARACTER_CONSTANT>{
+	[^\[\]\'\\\nMmY]+       ACTION_ECHO;
+        \'              ACTION_ECHO; yy_pop_state();
+}
+<ACTION_STRING,CHARACTER_CONSTANT>{
 	\\.		ACTION_ECHO;
 	{NL}		++linenum; ACTION_ECHO;
-	\"		ACTION_ECHO; BEGIN(ACTION);
 	.		ACTION_ECHO;
 }
 
-<COMMENT,COMMENT_DISCARD,ACTION,ACTION_STRING><<EOF>>	{
+<COMMENT,COMMENT_DISCARD,ACTION,ACTION_STRING,CHARACTER_CONSTANT><<EOF>>	{
 			synerr( _( "EOF encountered inside an action" ) );
 			yyterminate();
 			}
@@ -969,10 +981,15 @@ nmstr[yyleng - 2 - end_is_ws] = '\0';  /* chop trailing brace */
 
 
 <SECT3>{
-    /* "M4"|"m4"|"YY" fprintf (yyout, "[[%s]]", yytext); */
+    "M4"|"m4"|"YY" {
+        if (no_section3_escape) {
+           ECHO;
+        } else
+           fprintf (yyout, "[[%s]]", yytext);
+    }
     {M4QSTART}  fwrite (escaped_qstart, 1, strlen(escaped_qstart) - 0, yyout);
     {M4QEND}    fwrite (escaped_qend, 1, strlen(escaped_qend) - 0, yyout);
-	[^\[\]\n]*(\n?) ECHO;
+	[^\[\]MmY\n]*(\n?) ECHO;
 	(.|\n)      ECHO;
 	<<EOF>>		{
            //fwrite(M4QEND, 1, strlen(M4QEND), yyout);
diff --git a/src/yylex.c b/src/yylex.c
index c3de1d5..521db7f 100644
--- a/src/yylex.c
+++ b/src/yylex.c
@@ -37,18 +37,19 @@
 
 
 /* yylex - scan for a regular expression token */
-
 extern char *yytext;
+extern FILE *yyout;
+bool no_section3_escape = false;
 int     yylex (void)
 {
 	int     toktype;
 	static int beglin = false;
 
-	if (eofseen)
+	if (eofseen) {
 		toktype = EOF;
-	else
+        } else {
 		toktype = flexscan ();
-
+        }
 	if (toktype == EOF || toktype == 0) {
 		eofseen = 1;
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 00b7880..0c8df6a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -373,13 +373,13 @@ posixly_correct.c: posixly_correct.l $(FLEX)
 	POSIXLY_CORRECT=1 $(FLEX) -o $@ $<
 
 reject_nr.reject.c: reject.l4 $(FLEX)
-	$(FLEX) -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -o $@ $<
 
 reject_nr.reject$(EXEEXT): reject_nr.reject.$(OBJEXT)
 	$(LINK) $^
 
 reject_r.reject.c: reject.l4 $(FLEX)
-	$(FLEX) --reentrant -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape --reentrant -o $@ $<
 
 reject_r.reject.$(OBJEXT): reject_r.reject.c
 	$(COMPILE) -DTEST_IS_REENTRANT -c -o $@ $<
@@ -388,7 +388,7 @@ reject_r.reject$(EXEEXT): reject_r.reject.$(OBJEXT)
 	$(LINK) $^
 
 reject_ver.table.c: reject.l4 $(FLEX)
-	$(FLEX) -o $@ --tables-verify --tables-file=$(basename $@).tables $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -o $@ --tables-verify --tables-file=$(basename $@).tables $<
 
 reject_ver.table.$(OBJEXT): reject_ver.table.c
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
@@ -397,7 +397,7 @@ reject_ver.table$(EXEEXT): reject_ver.table.$(OBJEXT)
 	$(LINK) $^
 
 reject_ser.table.c: reject.l4 $(FLEX)
-	$(FLEX) -o $@ --tables-file=$(basename $@).tables $<
+	$(FLEX) -o $@ --unsafe-no-m4-sect3-escape --tables-file=$(basename $@).tables $<
 
 reject_ser.table.$(OBJEXT): reject_ser.table.c
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
@@ -431,13 +431,13 @@ OPT_LOG_COMPILER = $(srcdir)/testwrapper.sh
 AM_OPT_LOG_FLAGS = -d $(srcdir) -i $(srcdir)/tableopts.txt -r
 
 tableopts_opt_nr%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) $* -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) $* -o $@ $<
 
 tableopts_opt_nr%.$(OBJEXT): tableopts_opt_nr%.c 
 	$(COMPILE) -c -o $@ $<
 
 tableopts_opt_r%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) --reentrant $*  -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) --reentrant $*  -o $@ $<
 
 tableopts_opt_r%.$(OBJEXT):  tableopts_opt_r%.c 
 	$(COMPILE) -DTEST_IS_REENTRANT -c -o $@ $<
@@ -446,13 +446,13 @@ SER_LOG_COMPILER = $(srcdir)/testwrapper.sh
 AM_SER_LOG_FLAGS = -d $(builddir) -i $(srcdir)/tableopts.txt -r -t
 
 tableopts_ser_nr%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ser_nr$*.ser.tables"  $* -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ser_nr$*.ser.tables"  $* -o $@ $<
 
 tableopts_ser_nr%.$(OBJEXT): tableopts_ser_nr%.c 
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
 
 tableopts_ser_r%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ser_r$*.ser.tables" $*  -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ser_r$*.ser.tables" $*  -o $@ $<
 
 tableopts_ser_r%.$(OBJEXT):  tableopts_ser_r%.c
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -DTEST_IS_REENTRANT -c -o $@ $<
@@ -461,7 +461,7 @@ VER_LOG_COMPILER = $(srcdir)/testwrapper.sh
 AM_VER_LOG_FLAGS = -d $(builddir) -i $(srcdir)/tableopts.txt -r -t
 
 tableopts_ver_nr%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ver_nr$*.ver.tables" --tables-verify $* -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) --tables-file="tableopts_ver_nr$*.ver.tables" --tables-verify $* -o $@ $<
 
 tableopts_ver_nr%.$(OBJEXT): tableopts_ver_nr%.c 
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -c -o $@ $<
@@ -470,7 +470,7 @@ tableopts_ver_nr%.ver$(EXEEXT): tableopts_ver_nr%.$(OBJEXT)
 	$(LINK) -o $@ $^
 
 tableopts_ver_r%.c: tableopts.l4 $(FLEX)
-	$(FLEX) -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ver_r$*.ver.tables" --tables-verify $*  -o $@ $<
+	$(FLEX) --unsafe-no-m4-sect3-escape -P $(subst -,_,$(basename $(*F))) -R --tables-file="tableopts_ver_r$*.ver.tables" --tables-verify $*  -o $@ $<
 
 tableopts_ver_r%.$(OBJEXT):  tableopts_ver_r%.c  
 	$(COMPILE) -DTEST_HAS_TABLES_EXTERNAL -DTEST_IS_REENTRANT -c -o $@ $<
diff --git a/tests/alloc_extra.l b/tests/alloc_extra.l
index ae2ae99..c974777 100644
--- a/tests/alloc_extra.l
+++ b/tests/alloc_extra.l
@@ -75,8 +75,8 @@ main (void)
     testset_in(stdin, scanner);
     testset_out(stdout, scanner);
 
-    /* Test to confirm that yyalloc was called from
-     * yylex_init_extra with the yyextra argument.
+    /* Test to confirm that testalloc was called from
+     * testlex_init_extra with the testextra argument.
      */
     check_extra(scanner);
 
@@ -86,7 +86,7 @@ main (void)
     return 0;
 }
 
-void *yyalloc(size_t size, yyscan_t scanner)
+void *testalloc(size_t size, yyscan_t scanner)
 {
     struct Check *check;
     check = testget_extra(scanner);
diff --git a/tests/array_r.l b/tests/array_r.l
index 68a6299..e038f7c 100644
--- a/tests/array_r.l
+++ b/tests/array_r.l
@@ -49,13 +49,13 @@ main (void)
 {
     yyscan_t lexer;
     
-	yylex_init(&lexer);
-    yyset_in(stdin, lexer);
-    yyset_out(stdout, lexer);
+	testlex_init(&lexer);
+    testset_in(stdin, lexer);
+    testset_out(stdout, lexer);
 	
-    yylex( lexer );
+    testlex( lexer );
 	
-    yylex_destroy( lexer);
+    testlex_destroy( lexer);
     printf("TEST RETURNING OK.\n");
 
     return 0;
diff --git a/tests/basic_r.l b/tests/basic_r.l
index ba5734c..43d3a88 100644
--- a/tests/basic_r.l
+++ b/tests/basic_r.l
@@ -55,13 +55,13 @@ int main(void);
 int main (void)
 {
     yyscan_t  lexer;
-    yylex_init( &lexer );
-    yyset_out ( stdout,lexer);
-    yyset_in  ( stdin, lexer);
-    while( yylex(lexer) )
+    testlex_init( &lexer );
+    testset_out ( stdout,lexer);
+    testset_in  ( stdin, lexer);
+    while( testlex(lexer) )
     {
     }
-    yylex_destroy( lexer );
+    testlex_destroy( lexer );
     printf("TEST RETURNING OK.\n");
     return 0;
 }
diff --git a/tests/c_cxx_nr.lll b/tests/c_cxx_nr.lll
index c9e0fb6..24a73d7 100644
--- a/tests/c_cxx_nr.lll
+++ b/tests/c_cxx_nr.lll
@@ -48,10 +48,10 @@ int main(void);
 int
 main ()
 {
-    yyin = stdin;
-    yyout = stdout;
-    yylex();
-    yylex_destroy();
+    testin = stdin;
+    testout = stdout;
+    testlex();
+    testlex_destroy();
     printf("TEST RETURNING OK.\n");
     return 0;
 }
diff --git a/tests/c_cxx_r.lll b/tests/c_cxx_r.lll
index 83a39d7..e1fe9da 100644
--- a/tests/c_cxx_r.lll
+++ b/tests/c_cxx_r.lll
@@ -49,13 +49,13 @@ int
 main ()
 {
     yyscan_t  lexer;
-    yylex_init( &lexer );
-    yyset_out ( stdout,lexer);
-    yyset_in  ( stdin, lexer);
-    while( yylex(lexer) )
+    testlex_init( &lexer );
+    testset_out ( stdout,lexer);
+    testset_in  ( stdin, lexer);
+    while( testlex(lexer) )
     {
     }
-    yylex_destroy( lexer );
+    testlex_destroy( lexer );
     printf("TEST RETURNING OK.\n");
     return 0;
 }
diff --git a/tests/debug_r.l b/tests/debug_r.l
index 2734a54..8c2d7a5 100644
--- a/tests/debug_r.l
+++ b/tests/debug_r.l
@@ -43,17 +43,17 @@ int main(void);
 int main (void)
 {
     yyscan_t  lexer;
-    yylex_init( &lexer );
-    yyset_out ( stdout,lexer);
-    yyset_in  ( stdin, lexer);
+    testlex_init( &lexer );
+    testset_out ( stdout,lexer);
+    testset_in  ( stdin, lexer);
     
     /* Just see if the next line compiles. */
     testset_debug (testget_debug(lexer), lexer);
     
-    while( yylex(lexer) )
+    while( testlex(lexer) )
     {
     }
-    yylex_destroy( lexer );
+    testlex_destroy( lexer );
     printf("TEST RETURNING OK.\n");
     return 0;
 }
diff --git a/tests/include_by_reentrant.direct.l b/tests/include_by_reentrant.direct.l
index 2be8b38..7dbad72 100644
--- a/tests/include_by_reentrant.direct.l
+++ b/tests/include_by_reentrant.direct.l
@@ -59,11 +59,11 @@ int error = 0;
         error = 1;
         yyterminate();
     }
-    yylex_init(&scanner);
-    yyset_in( fp, scanner);
-    yyset_out( stdout, scanner);
-    yylex(scanner);
-    yylex_destroy(scanner);
+    testlex_init(&scanner);
+    testset_in( fp, scanner);
+    testset_out( stdout, scanner);
+    testlex(scanner);
+    testlex_destroy(scanner);
 
     BEGIN(0);
     }
@@ -93,11 +93,11 @@ main ( int argc, char **argv )
         fprintf(stderr,"*** Error: fopen(%s) failed.\n",argv[1]);
         exit(-1);
     }
-    yylex_init(&scanner);
-    yyset_in( fp, scanner);
-    yyset_out( stdout, scanner);
-    yylex(scanner);
-    yylex_destroy(scanner);
+    testlex_init(&scanner);
+    testset_in( fp, scanner);
+    testset_out( stdout, scanner);
+    testlex(scanner);
+    testlex_destroy(scanner);
     if (!error)
         printf("TEST RETURNING OK.\n");
     else
diff --git a/tests/lineno_r.l b/tests/lineno_r.l
index afd4874..d7230d7 100644
--- a/tests/lineno_r.l
+++ b/tests/lineno_r.l
@@ -87,11 +87,11 @@ main (int argc, char **argv)
 
     else{
         yyscan_t s;
-        yylex_init(&s);
-        yyset_in(stdin,s);
-        yyset_out(stdout,s);
-        yylex(s);
-        yylex_destroy(s);
+        testlex_init(&s);
+        testset_in(stdin,s);
+        testset_out(stdout,s);
+        testlex(s);
+        testlex_destroy(s);
     }
     return 0;
 }
diff --git a/tests/mem_nr.l b/tests/mem_nr.l
index 0c75dbd..b7cf343 100644
--- a/tests/mem_nr.l
+++ b/tests/mem_nr.l
@@ -166,8 +166,8 @@ main (void)
 
     yyin = stdin;
     yyout = stdout;
-    yylex();
-    yylex_destroy();
+    testlex();
+    testlex_destroy();
     free(ptrs);
 
     if ( nptrs > 0 || total_mem > 0){
diff --git a/tests/mem_r.l b/tests/mem_r.l
index 4b24294..06ae2e7 100644
--- a/tests/mem_r.l
+++ b/tests/mem_r.l
@@ -85,7 +85,7 @@ static void dump_mem(FILE* fp){
     fprintf(fp,"}\n");
 }
 
-void * yyalloc(yy_size_t n , void* yyscanner)
+void * testalloc(yy_size_t n , void* yyscanner)
 {
     (void)yyscanner;
 
@@ -116,7 +116,7 @@ void * yyalloc(yy_size_t n , void* yyscanner)
     return p;
 }
 
-void * yyrealloc(void* p, yy_size_t n , void* yyscanner)
+void * testrealloc(void* p, yy_size_t n , void* yyscanner)
 {
     (void)yyscanner;
 
@@ -139,7 +139,7 @@ void * yyrealloc(void* p, yy_size_t n , void* yyscanner)
     exit(1);
 }
 
-void yyfree(void* p , void* yyscanner)
+void testfree(void* p , void* yyscanner)
 {
     (void)yyscanner;
 
@@ -171,11 +171,11 @@ main (void)
     ptrs  = calloc(1, sizeof(struct memsz));
     nptrs = 0;
 
-    yylex_init(&scanner);
-    yyset_in(stdin,scanner);
-    yyset_out(stdout,scanner);
-    yylex(scanner);
-    yylex_destroy(scanner);
+    testlex_init(&scanner);
+    testset_in(stdin,scanner);
+    testset_out(stdout,scanner);
+    testlex(scanner);
+    testlex_destroy(scanner);
     free(ptrs);
 
     if ( nptrs > 0 || total_mem > 0){
diff --git a/tests/posix.l b/tests/posix.l
index a565a9d..f1851d8 100644
--- a/tests/posix.l
+++ b/tests/posix.l
@@ -63,9 +63,9 @@ int main (void)
 
     /* Run the tests */
     for (i=0; i < NUM_TESTS; i++){
-        printf("Testing: yy_scan_string(%s): ", tests[i]);
-        state = yy_scan_string(tests[i]);
-        yylex();
+        printf("Testing: test_scan_string(%s): ", tests[i]);
+        state = test_scan_string(tests[i]);
+        testlex();
         yy_delete_buffer(state);
         printf("... %s\n", tests_ok[i] ? "OK" : "FAILED");
     }
diff --git a/tests/posixly_correct.l b/tests/posixly_correct.l
index 75c3c91..715bb27 100644
--- a/tests/posixly_correct.l
+++ b/tests/posixly_correct.l
@@ -63,9 +63,9 @@ int main (void)
 
     /* Run the tests */
     for (i=0; i < NUM_TESTS; i++){
-        printf("Testing: yy_scan_string(%s): ", tests[i]);
-        state = yy_scan_string(tests[i]);
-        yylex();
+        printf("Testing: test_scan_string(%s): ", tests[i]);
+        state = test_scan_string(tests[i]);
+        testlex();
         yy_delete_buffer(state);
         printf("... %s\n", tests_ok[i] ? "OK" : "FAILED");
     }
diff --git a/tests/pthread.l b/tests/pthread.l
index 7d0dc7f..3f449d2 100644
--- a/tests/pthread.l
+++ b/tests/pthread.l
@@ -78,14 +78,14 @@ static int process_text(char* s, yyscan_t  scanner);
 <INITIAL,STATE_1,STATE_2>[[:space:]\r\n]+  { }
 %%
 
-int yywrap( yyscan_t  scanner) {
+int testwrap( yyscan_t  scanner) {
     (void)scanner;
     return 1;
 }
 static int process_text(char* s, yyscan_t  scanner)
 {
     (void)scanner;
-    return (int)(*s) + (int) *(s + yyget_leng(scanner)-1);
+    return (int)(*s) + (int) *(s + testget_leng(scanner)-1);
 }
 
 int main(int ARGC, char *ARGV[]);
@@ -136,19 +136,19 @@ static void * thread_func ( void* arg )
 
         pthread_mutex_lock ( &file_locks[ next ] );
 
-        yylex_init( &scanner );
+        testlex_init( &scanner );
         /*printf("Scanning file %s  #%d\n",filenames[next],i); fflush(stdout); */
         if((fp = fopen(filenames[next],"r"))==NULL) {
             perror("fopen");
             return NULL;
         }
-        yyset_in(fp,scanner);
+        testset_in(fp,scanner);
 
-        while( yylex( scanner) != 0)
+        while( testlex( scanner) != 0)
         {
         }
         fclose(fp);
-        yylex_destroy(scanner);
+        testlex_destroy(scanner);
         pthread_mutex_unlock ( &file_locks[ next ] );
     }
     return NULL;
diff --git a/tests/quotes.l b/tests/quotes.l
index c2bf2f0..618f1c1 100644
--- a/tests/quotes.l
+++ b/tests/quotes.l
@@ -38,6 +38,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include "config.h"
+#include <assert.h>
 /*#include "parser.h" */
 
 /* sect 1 block    [ 1 ]        TEST_XXX */
@@ -74,6 +75,7 @@ f       return 1+foo(a[b[c[0]]]);
             /* action block  [[[ 3 ]]]      TEST_XXX */
             /* action block [[[[ 4 ]]]]     TEST_XXX */
             /* action block ]] unmatched [[ TEST_XXX */
+            assert(!strcmp("m4_define(alpha, beta)", "m4_""define(alpha, beta)"));
             return 1+foo(a[b[c[0]]]);  //   TEST_XXX
          }
 %%
@@ -88,13 +90,17 @@ static int bar (int i){
 }
 int main(void);
 
+#define CONCAT_IDENTS(a, b) a##b
 int
 main (void)
 {
-    yyin = stdin;
+    // m4_m4exit(100)
+    FILE *M4_YY_NOT_IN_HEADER = stdin;
+    yyin = CONCAT_IDENTS(M4_, YY_NOT_IN_HEADER);
     yyout = stdout;
     while (yylex())
         ;
+    assert(!strcmp("YY_G( alpha)", "Y""Y_G( alpha)"));
     printf("TEST RETURNING OK.\n");
     return bar(0);
 }
diff --git a/tests/reject.l4 b/tests/reject.l4
index 7eda4a5..9bcde22 100644
--- a/tests/reject.l4
+++ b/tests/reject.l4
@@ -47,7 +47,7 @@ int main ( int argc, char** argv )
     M4_YY_DECL_GUTS_VAR();
 
 #ifdef TEST_IS_REENTRANT
-    yylex_init(&yyscanner);
+    testlex_init(&yyscanner);
 #else
     (void)yyscanner;
 #endif
@@ -67,13 +67,13 @@ int main ( int argc, char** argv )
             YY_FATAL_ERROR("could not open input file for reading");
         yyin = fp;
     }
-    while(yylex(M4_YY_CALL_ONLY_ARG) != 0)
+    while(testlex(M4_YY_CALL_ONLY_ARG) != 0)
         ;
         
 #ifdef TEST_HAS_TABLES_EXTERNAL
-    yytables_destroy(M4_YY_CALL_ONLY_ARG);
+    testtables_destroy(M4_YY_CALL_ONLY_ARG);
 #endif
-    yylex_destroy(M4_YY_CALL_ONLY_ARG);
+    testlex_destroy(M4_YY_CALL_ONLY_ARG);
 
     if(argc < 0) /* silence the compiler */
         yyscanner = (void*)fp;
diff --git a/tests/rescan_nr.direct.l b/tests/rescan_nr.direct.l
index 86103aa..536ba78 100644
--- a/tests/rescan_nr.direct.l
+++ b/tests/rescan_nr.direct.l
@@ -58,14 +58,14 @@ main (int argc, char* const argv[])
         return 1;
     }
 
-    yyset_out ( stdout);
+    testset_out ( stdout);
 
     for (i=0; i <  4; ++i){
         rewind(fp);
-        yyset_in  ( fp);
-        while( yylex() )
+        testset_in  ( fp);
+        while( testlex() )
             ;
-        yylex_destroy();
+        testlex_destroy();
     }
     printf("TEST RETURNING OK.\n");
     return 0;
diff --git a/tests/rescan_r.direct.l b/tests/rescan_r.direct.l
index 2255ee2..ceebc40 100644
--- a/tests/rescan_r.direct.l
+++ b/tests/rescan_r.direct.l
@@ -60,18 +60,18 @@ main (int argc, char* const argv[])
     }
 
     printf("Test 1: Reusing same scanner.\n");
-    yylex_init( &yyscanner );
-    yyset_out ( stdout, yyscanner);
+    testlex_init( &yyscanner );
+    testset_out ( stdout, yyscanner);
 
     for (i=0; i <  4; ++i){
 
         rewind(fp);
-        yyset_in  ( fp, yyscanner);
+        testset_in  ( fp, yyscanner);
 
-        while( yylex(yyscanner) )
+        while( testlex(yyscanner) )
             ;
     }
-    yylex_destroy( yyscanner );
+    testlex_destroy( yyscanner );
     printf("Test 1 OK\n\n");
 
     printf("Test 2: Rescanning with new scanner each time.\n");
@@ -80,14 +80,14 @@ main (int argc, char* const argv[])
 
     for (i=0; i < 4; ++i){
         yyscan_t  s;
-        yylex_init( &s );
-        yyset_out ( stdout, s);
+        testlex_init( &s );
+        testset_out ( stdout, s);
         rewind(fp);
-        yyset_in  ( fp, s);
+        testset_in  ( fp, s);
 
-        while( yylex(s) )
+        while( testlex(s) )
             ;
-        yylex_destroy( s );
+        testlex_destroy( s );
     }
     printf("Test 2 OK\n\n");
 
diff --git a/tests/string_nr.l b/tests/string_nr.l
index 26de54c..8352b09 100644
--- a/tests/string_nr.l
+++ b/tests/string_nr.l
@@ -67,29 +67,29 @@ main (void)
 
 
     /* Scan a good string. */
-    printf("Testing: yy_scan_string(%s): ",INPUT_STRING_1); fflush(stdout);
-    state = yy_scan_string ( INPUT_STRING_1 );
-    yylex();
+    printf("Testing: test_scan_string(%s): ",INPUT_STRING_1); fflush(stdout);
+    state = test_scan_string ( INPUT_STRING_1 );
+    testlex();
     yy_delete_buffer(state);
 
     /* Scan only the first 12 chars of a string. */
-    printf("Testing: yy_scan_bytes(%s): ",INPUT_STRING_2); fflush(stdout);
-    state = yy_scan_bytes  ( INPUT_STRING_2, 12 );
-    yylex();
-    yy_delete_buffer(state);
+    printf("Testing: test_scan_bytes(%s): ",INPUT_STRING_2); fflush(stdout);
+    state = test_scan_bytes  ( INPUT_STRING_2, 12 );
+    testlex();
+    test_delete_buffer(state);
 
     /* Scan directly from a buffer.
        We make a copy, since the buffer will be modified by flex.*/
-    printf("Testing: yy_scan_buffer(%s): ",INPUT_STRING_1); fflush(stdout);
+    printf("Testing: test_scan_buffer(%s): ",INPUT_STRING_1); fflush(stdout);
     len = strlen(INPUT_STRING_1) + 2;
     buf = malloc(len);
     strcpy( buf, INPUT_STRING_1);
     buf[ len -2 ]  = 0; /* Flex requires two NUL bytes at end of buffer. */
     buf[ len -1 ] =0;
 
-    state = yy_scan_buffer( buf, len );
-    yylex();
-    yy_delete_buffer(state);
+    state = test_scan_buffer( buf, len );
+    testlex();
+    test_delete_buffer(state);
     
     printf("TEST RETURNING OK.\n");
     return 0;
diff --git a/tests/string_r.l b/tests/string_r.l
index 392ba78..174eb6c 100644
--- a/tests/string_r.l
+++ b/tests/string_r.l
@@ -68,35 +68,35 @@ main (void)
 
 
     /* Scan a good string. */
-    printf("Testing: yy_scan_string(%s): ",INPUT_STRING_1); fflush(stdout);
-    yylex_init(&scanner);
-    state = yy_scan_string ( INPUT_STRING_1 ,scanner);
-    yylex(scanner);
-    yy_delete_buffer(state, scanner);
-    yylex_destroy(scanner);
+    printf("Testing: test_scan_string(%s): ",INPUT_STRING_1); fflush(stdout);
+    testlex_init(&scanner);
+    state = test_scan_string ( INPUT_STRING_1 ,scanner);
+    testlex(scanner);
+    test_delete_buffer(state, scanner);
+    testlex_destroy(scanner);
 
     /* Scan only the first 12 chars of a string. */
-    printf("Testing: yy_scan_bytes(%s): ",INPUT_STRING_2); fflush(stdout);
-    yylex_init(&scanner);
-    state = yy_scan_bytes  ( INPUT_STRING_2, 12 ,scanner);
-    yylex(scanner);
-    yy_delete_buffer(state,scanner);
-    yylex_destroy(scanner);
+    printf("Testing: test_scan_bytes(%s): ",INPUT_STRING_2); fflush(stdout);
+    testlex_init(&scanner);
+    state = test_scan_bytes  ( INPUT_STRING_2, 12 ,scanner);
+    testlex(scanner);
+    test_delete_buffer(state,scanner);
+    testlex_destroy(scanner);
 
     /* Scan directly from a buffer.
        We make a copy, since the buffer will be modified by flex.*/
-    printf("Testing: yy_scan_buffer(%s): ",INPUT_STRING_1); fflush(stdout);
+    printf("Testing: test_scan_buffer(%s): ",INPUT_STRING_1); fflush(stdout);
     len = strlen(INPUT_STRING_1) + 2;
     buf = malloc(len);
     strcpy( buf, INPUT_STRING_1);
     buf[ len -2 ]  = 0; /* Flex requires two NUL bytes at end of buffer. */
     buf[ len -1 ] =0;
 
-    yylex_init(&scanner);
-    state = yy_scan_buffer( buf, len ,scanner);
-    yylex(scanner);
-    yy_delete_buffer(state,scanner);
-    yylex_destroy(scanner);
+    testlex_init(&scanner);
+    state = test_scan_buffer( buf, len ,scanner);
+    testlex(scanner);
+    test_delete_buffer(state,scanner);
+    testlex_destroy(scanner);
     
     printf("TEST RETURNING OK.\n");
     return 0;