Added tests for overriding memory.

7 files changed, 483 insertions, 0 deletions

diff --git a/tests/test-mem-nr/.cvsignore b/tests/test-mem-nr/.cvsignore
new file mode 100644
index 0000000..7b73f56
--- /dev/null
+++ b/tests/test-mem-nr/.cvsignore
@@ -0,0 +1,9 @@
+Makefile
+Makefile.in
+parser.c
+parser.h
+scanner.c
+TEMPLATE
+OUTPUT
+.deps
+test-mem-nr
diff --git a/tests/test-mem-nr/Makefile.am b/tests/test-mem-nr/Makefile.am
new file mode 100644
index 0000000..191d9c4
--- /dev/null
+++ b/tests/test-mem-nr/Makefile.am
@@ -0,0 +1,44 @@
+# This file is part of flex.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+
+# Neither the name of the University nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE.
+
+BISON = @BISON@
+FLEX = $(top_builddir)/flex
+
+EXTRA_DIST = scanner.l test.input
+CLEANFILES = scanner.c scanner.h test-mem-nr OUTPUT $(OBJS)
+OBJS = scanner.o
+
+INCLUDES = -I $(srcdir) -I $(top_srcdir) -I $(top_builddir) -I .
+#LDFLAGS = $(top_srcdir)/libfl.a 
+
+testname = test-mem-nr
+
+scanner.c: $(srcdir)/scanner.l
+	$(FLEX) $(LFLAGS) $<
+
+$(testname)$(EXEEXT): $(OBJS)
+	$(CC) -o $@ $(OBJS) $(LDFLAGS)
+
+test: $(testname)$(EXEEXT)
+	./$(testname)$(EXEEXT) < $(srcdir)/test.input
+
+.c.o:
+	$(CC) -c -o $@ $(INCLUDES) $<
diff --git a/tests/test-mem-nr/scanner.l b/tests/test-mem-nr/scanner.l
new file mode 100644
index 0000000..8031ba6
--- /dev/null
+++ b/tests/test-mem-nr/scanner.l
@@ -0,0 +1,175 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+
+%{
+/* A template scanner file to build "scanner.c".
+ * The whole idea is to cause memory realloc by
+ *  1. pushing a lot on the condition stack, and
+ *  2. eating input greater than YY_BUF_SIZE
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include "config.h"
+
+/* Insanely small read buffer. This pretty much guarantees at least one realloc. */
+#define YY_BUF_SIZE 8
+
+%}
+
+%option 8bit outfile="scanner.c" prefix="test"
+%option nounput nomain noyywrap
+%option warn stack nodefault
+%option noyy_flex_alloc noyy_flex_realloc noyy_flex_free
+
+%x parens
+
+%%
+
+<INITIAL>{
+"("         { printf("yy_push_state(parens)\n"); yy_push_state(parens); }
+len=[0-9]+  { printf("About read token where %s\n",yytext); }
+0+          { }
+.|\n        { }
+}
+
+<parens>{
+"("         { printf("yy_push_state(parens)\n"); yy_push_state(parens); }
+")"         { printf("yy_pop_state()\n");yy_pop_state();}
+[^()\n]+     { }
+.|\n        { }
+}
+
+%%
+/* total memory allocated */
+static size_t total_mem=0;
+
+/* track the amount of memory for ptr. */
+struct memsz {
+    void* p;
+    size_t sz;
+};
+
+static struct memsz * ptrs=0;  /* Array of pairs. */
+static int nptrs=0; /* Number of pairs in array. */
+static int arrsz=0; /* Capacity of array. */
+
+static void dump_mem(FILE* fp){
+    int i;
+    fprintf(fp,"\tptrs[%d] = {", nptrs);
+    for (i=0; i < arrsz; i++)
+        fprintf(fp," {%#lx,%d},", (long)ptrs[i].p, (long)ptrs[i].sz);
+    
+    fprintf(fp,"}\n");
+}
+
+void * yy_flex_alloc(size_t n YY_LAST_ARG)
+{
+    void * p;
+    struct memsz * old;
+    int i;
+
+    total_mem += n;
+    p = (void*)malloc(n);
+
+    if( nptrs >= arrsz){
+        /* increase array size by 1 */
+        arrsz++;
+        ptrs = (struct memsz*)realloc( ptrs, arrsz * sizeof(struct memsz));
+        ptrs[nptrs].p = 0;
+        ptrs[nptrs].sz = 0;
+    }
+
+    /* find a null slot */
+    for(i=0; i < arrsz ; i++)
+        if (ptrs[i].p == 0) {
+            ptrs[i].p = p;
+            ptrs[i].sz = n;
+        }
+
+    nptrs++;
+    printf("yyflex_alloc(%8ld) total=%8ld return=%#10lx\n",(long)n,(long)total_mem,(long)p);
+    dump_mem(stdout);
+    return p;
+}
+
+void * yy_flex_realloc(void* p, size_t n YY_LAST_ARG)
+{
+    int i;
+    for (i=0; i < arrsz; i++)
+        if ( ptrs[i].p == p){
+            total_mem -= ptrs[i].sz;
+            total_mem += n;
+            ptrs[i].p = (void*)realloc(p,n);
+            ptrs[i].sz = n;
+
+            printf("yyflex_realloc(%#10lx,%8ld) total=%8ld return=%8lx\n",
+                    (long)p,n,(long)total_mem,(long)ptrs[i].p);
+            dump_mem(stdout);
+            return ptrs[i].p;
+        }
+
+    fprintf(stderr,"ERROR: yyflex_realloc could not locate pointer %#lx.\n",(long)p);
+    dump_mem(stdout);
+    exit(1);
+}
+
+void yy_flex_free(void* p YY_LAST_ARG)
+{
+    int i;
+    for (i=0; i < arrsz; i++)
+        if ( ptrs[i].p == p){
+            total_mem -= ptrs[i].sz;
+            free(p);
+            ptrs[i].p = 0;
+            ptrs[i].sz = 0;
+            nptrs--;
+            printf("yyflex_free(%#10lx) total=%8ld\n",(long)p,(long)total_mem);
+            dump_mem(stdout);
+            return;
+        }
+
+    fprintf(stderr,"ERROR: yyflex_free could not locate pointer %#lx.\n",(long)p);
+    dump_mem(stdout);
+    exit(1);
+}
+
+int
+main ( int argc, char** argv )
+{
+    arrsz = 1;
+    ptrs  = (struct memsz*)calloc(1,sizeof(struct memsz));
+    nptrs = 0;
+
+    yyin = stdin;
+    yyout = stdout;
+    yylex();
+    yylex_destroy();
+    free(ptrs);
+
+    if ( nptrs > 0 || total_mem > 0){
+        fprintf(stderr,"Oops. Looks like a memory leak: nptrs=%d, unfreed memory=%ld\n",nptrs,(long)total_mem);
+        exit(1);
+    }
+    printf("TEST RETURNING OK.\n");
+    return 0;
+}
diff --git a/tests/test-mem-r/.cvsignore b/tests/test-mem-r/.cvsignore
new file mode 100644
index 0000000..a21d062
--- /dev/null
+++ b/tests/test-mem-r/.cvsignore
@@ -0,0 +1,9 @@
+Makefile
+Makefile.in
+parser.c
+parser.h
+scanner.c
+TEMPLATE
+OUTPUT
+.deps
+test-mem-r
diff --git a/tests/test-mem-r/Makefile.am b/tests/test-mem-r/Makefile.am
new file mode 100644
index 0000000..c3499ee
--- /dev/null
+++ b/tests/test-mem-r/Makefile.am
@@ -0,0 +1,44 @@
+# This file is part of flex.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+
+# Neither the name of the University nor the names of its contributors
+# may be used to endorse or promote products derived from this software
+# without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE.
+
+BISON = @BISON@
+FLEX = $(top_builddir)/flex
+
+EXTRA_DIST = scanner.l test.input
+CLEANFILES = scanner.c scanner.h test-mem-r OUTPUT $(OBJS)
+OBJS = scanner.o
+
+INCLUDES = -I $(srcdir) -I $(top_srcdir) -I $(top_builddir) -I .
+#LDFLAGS = $(top_srcdir)/libfl.a 
+
+testname = test-mem-r
+
+scanner.c: $(srcdir)/scanner.l
+	$(FLEX) $(LFLAGS) $<
+
+$(testname)$(EXEEXT): $(OBJS)
+	$(CC) -o $@ $(OBJS) $(LDFLAGS)
+
+test: $(testname)$(EXEEXT)
+	./$(testname)$(EXEEXT) < $(srcdir)/test.input
+
+.c.o:
+	$(CC) -c -o $@ $(INCLUDES) $<
diff --git a/tests/test-mem-r/scanner.l b/tests/test-mem-r/scanner.l
new file mode 100644
index 0000000..d4ba2e1
--- /dev/null
+++ b/tests/test-mem-r/scanner.l
@@ -0,0 +1,177 @@
+/*
+ * This file is part of flex.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE.
+ */
+
+%{
+/* A template scanner file to build "scanner.c".
+ * The whole idea is to cause memory realloc by
+ *  1. pushing a lot on the condition stack, and
+ *  2. eating input greater than YY_BUF_SIZE
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include "config.h"
+
+/* Insanely small read buffer. This pretty much guarantees at least one realloc. */
+#define YY_BUF_SIZE 8
+
+%}
+
+%option 8bit outfile="scanner.c" prefix="test"
+%option nounput nomain noyywrap
+%option warn stack nodefault reentrant
+%option noyy_flex_alloc noyy_flex_realloc noyy_flex_free
+
+%x parens
+
+%%
+
+<INITIAL>{
+"("         { printf("yy_push_state(parens)\n"); yy_push_state(parens,yy_globals); }
+len=[0-9]+  { printf("About read token where %s\n",yytext); }
+0+          { }
+.|\n        { }
+}
+
+<parens>{
+"("         { printf("yy_push_state(parens)\n"); yy_push_state(parens,yy_globals); }
+")"         { printf("yy_pop_state()\n");yy_pop_state(yy_globals);}
+[^()\n]+     { }
+.|\n        { }
+}
+
+%%
+/* total memory allocated */
+static size_t total_mem=0;
+
+/* track the amount of memory for ptr. */
+struct memsz {
+    void* p;
+    size_t sz;
+};
+
+static struct memsz * ptrs=0;  /* Array of pairs. */
+static int nptrs=0; /* Number of pairs in array. */
+static int arrsz=0; /* Capacity of array. */
+
+static void dump_mem(FILE* fp){
+    int i;
+    fprintf(fp,"\tptrs[%d] = {", nptrs);
+    for (i=0; i < arrsz; i++)
+        fprintf(fp," {%#lx,%d},", (long)ptrs[i].p, (long)ptrs[i].sz);
+    
+    fprintf(fp,"}\n");
+}
+
+void * yy_flex_alloc(size_t n YY_LAST_ARG)
+{
+    void * p;
+    struct memsz * old;
+    int i;
+
+    total_mem += n;
+    p = (void*)malloc(n);
+
+    if( nptrs >= arrsz){
+        /* increase array size by 1 */
+        arrsz++;
+        ptrs = (struct memsz*)realloc( ptrs, arrsz * sizeof(struct memsz));
+        ptrs[nptrs].p = 0;
+        ptrs[nptrs].sz = 0;
+    }
+
+    /* find a null slot */
+    for(i=0; i < arrsz ; i++)
+        if (ptrs[i].p == 0) {
+            ptrs[i].p = p;
+            ptrs[i].sz = n;
+        }
+
+    nptrs++;
+    printf("yyflex_alloc(%8ld) total=%8ld return=%#10lx\n",(long)n,(long)total_mem,(long)p);
+    dump_mem(stdout);
+    return p;
+}
+
+void * yy_flex_realloc(void* p, size_t n YY_LAST_ARG)
+{
+    int i;
+    for (i=0; i < arrsz; i++)
+        if ( ptrs[i].p == p){
+            total_mem -= ptrs[i].sz;
+            total_mem += n;
+            ptrs[i].p = (void*)realloc(p,n);
+            ptrs[i].sz = n;
+
+            printf("yyflex_realloc(%#10lx,%8ld) total=%8ld return=%8lx\n",
+                    (long)p,n,(long)total_mem,(long)ptrs[i].p);
+            dump_mem(stdout);
+            return ptrs[i].p;
+        }
+
+    fprintf(stderr,"ERROR: yyflex_realloc could not locate pointer %#lx.\n",(long)p);
+    dump_mem(stdout);
+    exit(1);
+}
+
+void yy_flex_free(void* p YY_LAST_ARG)
+{
+    int i;
+    for (i=0; i < arrsz; i++)
+        if ( ptrs[i].p == p){
+            total_mem -= ptrs[i].sz;
+            free(p);
+            ptrs[i].p = 0;
+            ptrs[i].sz = 0;
+            nptrs--;
+            printf("yyflex_free(%#10lx) total=%8ld\n",(long)p,(long)total_mem);
+            dump_mem(stdout);
+            return;
+        }
+
+    fprintf(stderr,"ERROR: yyflex_free could not locate pointer %#lx.\n",(long)p);
+    dump_mem(stdout);
+    exit(1);
+}
+
+int
+main ( int argc, char** argv )
+{
+    yyscan_t scanner;
+    arrsz = 1;
+    ptrs  = (struct memsz*)calloc(1,sizeof(struct memsz));
+    nptrs = 0;
+
+    yylex_init(&scanner);
+    yyset_in(stdin,scanner);
+    yyset_out(stdout,scanner);
+    yylex(scanner);
+    yylex_destroy(scanner);
+    free(ptrs);
+
+    if ( nptrs > 0 || total_mem > 0){
+        fprintf(stderr,"Oops. Looks like a memory leak: nptrs=%d, unfreed memory=%ld\n",nptrs,(long)total_mem);
+        exit(1);
+    }
+    printf("TEST RETURNING OK.\n");
+    return 0;
+}
diff --git a/tests/test-mem-r/test.input b/tests/test-mem-r/test.input
new file mode 100644
index 0000000..79aa16a
--- /dev/null
+++ b/tests/test-mem-r/test.input
@@ -0,0 +1,25 @@
+First we push a lot on the stack by nesting parenthesis:
+
+((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
+((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
+(((((((((((((((((((((((((((((((((((((((((((
+
+(should be 200 states pushed here)
+
+))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
+))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
+)))))))))))))))))))))))))))))))))))))))))))
+
+Now we match progressively bigger tokens to increase the read buffer:
+
+len=1    0
+len=2    00
+len=4    0000
+len=8    00000000
+len=16   0000000000000000
+len=32   00000000000000000000000000000000
+len=64   0000000000000000000000000000000000000000000000000000000000000000
+len=128  00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+len=256  0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+len=512  00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+len=1024 0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000