[brlcad-commits] SF.net SVN: brlcad:[47512] brlcad/trunk/src/other/perplex

[<n_...@us...>]

Revision: 47512
          http://brlcad.svn.sourceforge.net/brlcad/?rev=47512&view=rev
Author:   n_reed
Date:     2011-11-16 22:55:59 +0000 (Wed, 16 Nov 2011)
Log Message:
-----------
borrowing flex's dynamic buffers to implement yytext string

Modified Paths:
--------------
    brlcad/trunk/src/other/perplex/Makefile.local
    brlcad/trunk/src/other/perplex/perplex.c
    brlcad/trunk/src/other/perplex/perplex.h
    brlcad/trunk/src/other/perplex/scanner.re
    brlcad/trunk/src/other/perplex/template.c

Modified: brlcad/trunk/src/other/perplex/Makefile.local
===================================================================
--- brlcad/trunk/src/other/perplex/Makefile.local	2011-11-16 22:28:08 UTC (rev 47511)
+++ brlcad/trunk/src/other/perplex/Makefile.local	2011-11-16 22:55:59 UTC (rev 47512)
@@ -1,15 +1,13 @@
 CFLAGS = -g
 
 perplex : perplex.o scanner.o
-	$(CC) -o $@ $^
+	$(CC) -o $@ $^ -lm
 
-perplex.o : scanner.h
+scanner.c perplex.o : perplex.h
 
-scanner.c : scanner.re scanner.h
+scanner.c : scanner.re
 	re2c -c -o $@ $<
 
-scanner.h :
-
 .PHONY : clean
 
 clean :

Modified: brlcad/trunk/src/other/perplex/perplex.c
===================================================================
--- brlcad/trunk/src/other/perplex/perplex.c	2011-11-16 22:28:08 UTC (rev 47511)
+++ brlcad/trunk/src/other/perplex/perplex.c	2011-11-16 22:55:59 UTC (rev 47512)
@@ -1,3 +1,43 @@
+/*                       P E R P L E X . C
+ * BRL-CAD
+ *
+ * Copyright (c) 2011 United States Government as represented by
+ * the U.S. Army Research Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * 3. The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** @file perplex.c
+ *
+ * perplex scanner-generator
+ *
+ */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include "perplex.h"
@@ -71,3 +111,13 @@
 
     return 0;
 }
+
+/*
+ * Local Variables:
+ * tab-width: 8
+ * mode: C
+ * indent-tabs-mode: t
+ * c-file-style: "stroustrup"
+ * End:
+ * ex: shiftwidth=4 tabstop=8
+ */

Modified: brlcad/trunk/src/other/perplex/perplex.h
===================================================================
--- brlcad/trunk/src/other/perplex/perplex.h	2011-11-16 22:28:08 UTC (rev 47511)
+++ brlcad/trunk/src/other/perplex/perplex.h	2011-11-16 22:55:59 UTC (rev 47512)
@@ -1,3 +1,43 @@
+/*                       P E R P L E X . H
+ * BRL-CAD
+ *
+ * Copyright (c) 2011 United States Government as represented by
+ * the U.S. Army Research Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * 3. The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/** @file perplex.h
+ *
+ * definitions common to the perplex main and scanner sources
+ *
+ */
+
 #ifndef PERPLEX_H
 #define PERPLEX_H
 
@@ -29,9 +69,9 @@
     char *cursor;
     char *marker;
     char *null;
+    char *tokenStart;
     char *bufLast;
     char *buffer;
-    char *yytext;
     condition_t condition;
 } *perplex_t;
 
@@ -40,3 +80,13 @@
 void perplexFree(perplex_t scanner);
 
 #endif
+
+/*
+ * Local Variables:
+ * tab-width: 8
+ * mode: C
+ * indent-tabs-mode: t
+ * c-file-style: "stroustrup"
+ * End:
+ * ex: shiftwidth=4 tabstop=8
+ */

Modified: brlcad/trunk/src/other/perplex/scanner.re
===================================================================
--- brlcad/trunk/src/other/perplex/scanner.re	2011-11-16 22:28:08 UTC (rev 47511)
+++ brlcad/trunk/src/other/perplex/scanner.re	2011-11-16 22:55:59 UTC (rev 47512)
@@ -1,4 +1,51 @@
-/* The perplex input scanner. Converts flex-style input to re2c input.
+/*                        S C A N N E R . R E
+ * BRL-CAD
+ *
+ * Copyright (c) 1990-2011 United States Government as represented by
+ * the U.S. Army Research Laboratory.
+ *
+ * Copyright (c) 1990 The Regents of the University of California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * 3. The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*  Parts of this file are based on sources from the flex project.
+ *
+ *  This code is derived from software contributed to Berkeley by
+ *  Vern Paxson.
+ *
+ *  The United States Government has rights in this work pursuant
+ *  to contract no. DE-AC03-76SF00098 between the United States
+ *  Department of Energy and the University of California.
+ */
+/** @file scanner.re
+ *
+ * The perplex input scanner. Converts flex-style input to re2c input.
  * Usage:
  *     int tokenID;
  *     perplex_t scanner;
@@ -14,6 +61,8 @@
 #include <stdlib.h>
 #include "perplex.h"
 
+/* start-condition support */
+
 static void
 setCondition(perplex_t scanner, condition_t cond)
 {
@@ -26,6 +75,259 @@
     return scanner->condition;
 }
 
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define YYEOF -1
+
+/* --- from flex's flexdef.h --- */
+struct Buf {
+    void   *elts;	/* elements. */
+    int     nelts;	/* number of elements. */
+    size_t  elt_size;	/* in bytes. */
+    int     nmax;	/* max capacity of elements. */
+};
+
+void buf_init(struct Buf * buf, size_t elem_size);
+void buf_destroy(struct Buf * buf);
+struct Buf *buf_append(struct Buf * buf, const void *ptr, int n_elem);
+struct Buf *buf_concat(struct Buf* dest, const struct Buf* src);
+struct Buf *buf_strappend(struct Buf *, const char *str);
+struct Buf *buf_strnappend(struct Buf *, const char *str, int nchars);
+struct Buf *buf_strdefine(struct Buf * buf, const char *str, const char *def);
+struct Buf *buf_prints(struct Buf *buf, const char *fmt, const char* s);
+struct Buf *buf_m4_define(struct Buf *buf, const char* def, const char* val);
+struct Buf *buf_m4_undefine(struct Buf *buf, const char* def);
+struct Buf *buf_print_strings(struct Buf * buf, FILE* out);
+struct Buf *buf_linedir(struct Buf *buf, const char* filename, int lineno);
+
+/* --- from flex's misc.c --- */
+static void*
+allocate_array(int size, size_t element_size)
+{
+    return malloc(element_size * size);
+}
+
+static void*
+reallocate_array(void *array, int size, size_t element_size)
+{
+    return realloc(array, element_size * size);
+}
+
+/* --- from flex's buf.c --- */
+/* Take note: The buffer object is sometimes used as a String buffer (one
+ * continuous string), and sometimes used as a list of strings, usually line by
+ * line.
+ * 
+ * The type is specified in buf_init by the elt_size. If the elt_size is
+ * sizeof(char), then the buffer should be treated as string buffer. If the
+ * elt_size is sizeof(char*), then the buffer should be treated as a list of
+ * strings.
+ *
+ * Certain functions are only appropriate for one type or the other. 
+ */
+
+struct Buf*
+buf_print_strings(struct Buf * buf, FILE* out)
+{
+    int i;
+
+    if(!buf || !out) {
+        return buf;
+    }
+
+    for (i = 0; i < buf->nelts; i++) {
+        const char * s = ((char**)buf->elts)[i];
+        if(s) {
+            fprintf(out, "%s", s);
+	}
+    }
+    return buf;
+}
+
+/* Append a "%s" formatted string to a string buffer */
+struct Buf*
+buf_prints(struct Buf *buf, const char *fmt, const char *s)
+{
+    char *t;
+    size_t tsz;
+
+    t = malloc(tsz = strlen(fmt) + strlen(s) + 1);
+    snprintf(t, tsz, fmt, s);
+    buf = buf_strappend(buf, t);
+    free(t);
+    return buf;
+}
+
+/** Append a line directive to the string buffer.
+ * @param buf A string buffer.
+ * @param filename file name
+ * @param lineno line number
+ * @return buf
+ */
+struct Buf*
+buf_linedir(struct Buf *buf, const char* filename, int lineno)
+{
+    char   *t, *fmt = "#line %d \"%s\"\n";
+    size_t tsz;
+    
+    t = malloc(tsz = strlen(fmt) + strlen(filename) + (int)(1 + log10(lineno >= 0? lineno : -lineno)) + 1);
+    snprintf(t, tsz, fmt, lineno, filename);
+    buf = buf_strappend(buf, t);
+    free(t);
+    return buf;
+}
+
+
+/** Append the contents of @a src to @a dest.
+ * @param @a dest the destination buffer
+ * @param @a dest the source buffer
+ * @return @a dest
+ */
+struct Buf*
+buf_concat(struct Buf* dest, const struct Buf* src)
+{
+    buf_append(dest, src->elts, src->nelts);
+    return dest;
+}
+
+
+/* Appends n characters in str to buf. */
+struct Buf*
+buf_strnappend(struct Buf *buf, const char *str, int n)
+{
+    buf_append(buf, str, n + 1);
+
+    /* "undo" the '\0' character that buf_append() already copied. */
+    buf->nelts--;
+
+    return buf;
+}
+
+/* Appends characters in str to buf. */
+struct Buf*
+buf_strappend(struct Buf *buf, const char *str)
+{
+    return buf_strnappend(buf, str, strlen(str));
+}
+
+/* appends "#define str def\n" */
+struct Buf*
+buf_strdefine(struct Buf *buf, const char *str, const char *def)
+{
+    buf_strappend(buf, "#define ");
+    buf_strappend(buf, " ");
+    buf_strappend(buf, str);
+    buf_strappend(buf, " ");
+    buf_strappend(buf, def);
+    buf_strappend(buf, "\n");
+    return buf;
+}
+
+/** Pushes "m4_define( [[def]], [[val]])m4_dnl" to end of buffer.
+ * @param buf A buffer as a list of strings.
+ * @param def The m4 symbol to define.
+ * @param val The definition; may be NULL.
+ * @return buf
+ */
+struct Buf*
+buf_m4_define(struct Buf *buf, const char* def, const char* val)
+{
+    const char * fmt = "m4_define( [[%s]], [[%s]])m4_dnl\n";
+    char * str;
+    size_t strsz;
+
+    val = val?val:"";
+    str = (char*)malloc(strsz = strlen(fmt) + strlen(def) + strlen(val) + 2);
+
+    snprintf(str, strsz, fmt, def, val);
+    buf_append(buf, &str, 1);
+    return buf;
+}
+
+/** Pushes "m4_undefine([[def]])m4_dnl" to end of buffer.
+ * @param buf A buffer as a list of strings.
+ * @param def The m4 symbol to undefine.
+ * @return buf
+ */
+struct Buf*
+buf_m4_undefine(struct Buf *buf, const char* def)
+{
+    const char * fmt = "m4_undefine( [[%s]])m4_dnl\n";
+    char * str;
+    size_t strsz;
+
+    str = (char*)malloc(strsz = strlen(fmt) + strlen(def) + 2);
+
+    snprintf(str, strsz, fmt, def);
+    buf_append(buf, &str, 1);
+    return buf;
+}
+
+/* create buf with 0 elements, each of size elem_size. */
+void
+buf_init(struct Buf *buf, size_t elem_size)
+{
+    buf->elts = (void *) 0;
+    buf->nelts = 0;
+    buf->elt_size = elem_size;
+    buf->nmax = 0;
+}
+
+/* frees memory */
+void
+buf_destroy(struct Buf *buf)
+{
+    if (buf && buf->elts) {
+	free(buf->elts);
+    }
+    buf->elts = (void *) 0;
+}
+
+
+/* appends ptr[] to buf, grow if necessary.
+ * n_elem is number of elements in ptr[], NOT bytes.
+ * returns buf.
+ * We grow by mod(512) boundaries.
+ */
+struct Buf*
+buf_append(struct Buf *buf, const void *ptr, int n_elem)
+{
+    int n_alloc = 0;
+
+    if (!ptr || n_elem == 0) {
+	return buf;
+    }
+
+    /* May need to alloc more. */
+    if (n_elem + buf->nelts > buf->nmax) {
+	/* exact amount needed... */
+	n_alloc = (n_elem + buf->nelts) * buf->elt_size;
+
+	/* ...plus some extra */
+	if (((n_alloc * buf->elt_size) % 512) != 0 && buf->elt_size < 512) {
+	    n_alloc += (512 - ((n_alloc * buf->elt_size) % 512)) / buf->elt_size;
+	}
+	if (!buf->elts) {
+	    buf->elts = allocate_array(n_alloc, buf->elt_size);
+	} else {
+	    buf->elts = reallocate_array(buf->elts, n_alloc, buf->elt_size);
+	}
+	buf->nmax = n_alloc;
+    }
+    memcpy((char*)buf->elts + buf->nelts * buf->elt_size, ptr,
+	n_elem * buf->elt_size);
+
+    buf->nelts += n_elem;
+
+    return buf;
+}
+
+/* --- */
+/* input buffering support */
+
 /*!max:re2c*/
 
 /* Estimate of largest number of characters "in use" at once:
@@ -36,13 +338,13 @@
 /* Size of scanner buffer. Should be big enough to comfortably hold
  * everything from backtracking marker to null marker.
  */
-static const int BUF_SIZE = (MAX_IN_USE > 128) ? MAX_IN_USE : 128;
+static const int BUF_SIZE = MAX_IN_USE;
 
 /* Copy up to n input characters to the end of scanner buffer.
  * If EOF is encountered before n characters are read, '\0'
  * is appended to the buffer to serve as EOF indicator.
  */
-void
+static void
 bufferAppend(perplex_t scanner, size_t n)
 {
     FILE *in = scanner->in.file;
@@ -68,7 +370,7 @@
  * Buffer contents are shifted if there is insufficient room
  * at the end of the buffer.
  */
-void
+static void
 bufferFill(perplex_t scanner, size_t n)
 {
     size_t i, shiftSize;
@@ -93,6 +395,19 @@
     bufferAppend(scanner, n);
 }
 
+static char*
+getTokenText(perplex_t scanner, struct Buf *localBuf)
+{
+    char term = '\0';
+    int tokenChars = scanner->cursor - scanner->tokenStart;
+
+    localBuf->nelts = 0;
+    buf_append(localBuf, scanner->tokenStart, tokenChars);
+    buf_append(localBuf, &term, sizeof(term));
+
+    return (char*)localBuf->elts;
+}
+
 static perplex_t
 newScanner()
 {
@@ -142,21 +457,31 @@
     free(scanner);
 }
 
-#define RETURN(id) return id
-#define YYGETCONDITION getCondition(scanner)
-#define YYFILL(n) bufferFill(scanner, n)
+#define YYGETCONDITION  getCondition(scanner)
+#define YYFILL(n)       bufferFill(scanner, n)
 
-/* if it's re2c, echo it
- * otherwise, convert it to re2c
- */
+#define UPDATE_START  scanner->tokenStart = scanner->cursor;
+#define yytext        getTokenText(scanner, _perplex_token_string)
+#define RETURN(id)    buf_destroy(_perplex_token_string); return id;
+#define CONTINUE      UPDATE_START; continue;
+
+
 int yylex(perplex_t scanner) {
+    char yych;
+    struct Buf *_perplex_token_string;
 
+    UPDATE_START;
+
+    _perplex_token_string = (struct Buf*)malloc(sizeof(struct Buf));
+    buf_init(_perplex_token_string, sizeof(char));
+
     while (1) {
 /*!re2c
 re2c:condenumprefix = "";
 re2c:define:YYCONDTYPE = condition_t;
 re2c:define:YYGETCONDITION:naked = 1;
 
+re2c:yych:emit = 0;
 re2c:define:YYCTYPE  = char;
 re2c:define:YYCURSOR = scanner->cursor;
 re2c:define:YYLIMIT  = scanner->null;
@@ -174,9 +499,9 @@
 <*>EOF { RETURN(YYEOF); }
 
 <*>FAUX_SEPARATOR {
-    /* print text as it appears */
-    printf("%c%%%%\n", scanner->cursor[-4]);
-    continue;
+    /* just text - echo it */
+    printf("%s\n", yytext);
+    CONTINUE;
 }
 
 <DEFINITIONS>SEPARATOR {
@@ -190,10 +515,20 @@
 }
 
 <*>ANY {
-    /* echo rule */
-    printf("%c", scanner->cursor[-1]);
-    continue;
+    /* just text - echo it */
+    printf("%s", yytext);
+    CONTINUE;
 }
 */
     }
 }
+
+/*
+ * Local Variables:
+ * tab-width: 8
+ * mode: C
+ * indent-tabs-mode: t
+ * c-file-style: "stroustrup"
+ * End:
+ * ex: shiftwidth=4 tabstop=8
+ */

Modified: brlcad/trunk/src/other/perplex/template.c
===================================================================
--- brlcad/trunk/src/other/perplex/template.c	2011-11-16 22:28:08 UTC (rev 47511)
+++ brlcad/trunk/src/other/perplex/template.c	2011-11-16 22:55:59 UTC (rev 47512)
@@ -1,5 +1,53 @@
-/*!max:re2c*/
-
+/*                        T E M P L A T E . C
+ * BRL-CAD
+ *
+ * Copyright (c) 1990-2011 United States Government as represented by
+ * the U.S. Army Research Laboratory.
+ *
+ * Copyright (c) 1990 The Regents of the University of California.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * 3. The name of the author may not be used to endorse or promote
+ * products derived from this software without specific prior written
+ * permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+ * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*  Parts of this file are based on sources from the flex project.
+ *
+ *  This code is derived from software contributed to Berkeley by
+ *  Vern Paxson.
+ *
+ *  The United States Government has rights in this work pursuant
+ *  to contract no. DE-AC03-76SF00098 between the United States
+ *  Department of Energy and the University of California.
+ */
+/** @file template.c
+ *
+ * template for generated scanners
+ *
+ */
 /* scanner data */
 typedef struct perplex_data_t {
     union {
@@ -9,13 +57,266 @@
     char *cursor;
     char *marker;
     char *null;
+    char *tokenStart;
     char *bufLast;
     char *buffer;
-    char *yytext;
 } *perplex_t;
 
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
 #define YYEOF -1
 
+/* --- from flex's flexdef.h --- */
+struct Buf {
+    void   *elts;	/* elements. */
+    int     nelts;	/* number of elements. */
+    size_t  elt_size;	/* in bytes. */
+    int     nmax;	/* max capacity of elements. */
+};
+
+void buf_init(struct Buf * buf, size_t elem_size);
+void buf_destroy(struct Buf * buf);
+struct Buf *buf_append(struct Buf * buf, const void *ptr, int n_elem);
+struct Buf *buf_concat(struct Buf* dest, const struct Buf* src);
+struct Buf *buf_strappend(struct Buf *, const char *str);
+struct Buf *buf_strnappend(struct Buf *, const char *str, int nchars);
+struct Buf *buf_strdefine(struct Buf * buf, const char *str, const char *def);
+struct Buf *buf_prints(struct Buf *buf, const char *fmt, const char* s);
+struct Buf *buf_m4_define(struct Buf *buf, const char* def, const char* val);
+struct Buf *buf_m4_undefine(struct Buf *buf, const char* def);
+struct Buf *buf_print_strings(struct Buf * buf, FILE* out);
+struct Buf *buf_linedir(struct Buf *buf, const char* filename, int lineno);
+
+/* --- from flex's misc.c --- */
+static void*
+allocate_array(int size, size_t element_size)
+{
+    return malloc(element_size * size);
+}
+
+static void*
+reallocate_array(void *array, int size, size_t element_size)
+{
+    return realloc(array, element_size * size);
+}
+
+/* --- from flex's buf.c --- */
+/* Take note: The buffer object is sometimes used as a String buffer (one
+ * continuous string), and sometimes used as a list of strings, usually line by
+ * line.
+ * 
+ * The type is specified in buf_init by the elt_size. If the elt_size is
+ * sizeof(char), then the buffer should be treated as string buffer. If the
+ * elt_size is sizeof(char*), then the buffer should be treated as a list of
+ * strings.
+ *
+ * Certain functions are only appropriate for one type or the other. 
+ */
+
+struct Buf*
+buf_print_strings(struct Buf * buf, FILE* out)
+{
+    int i;
+
+    if(!buf || !out) {
+        return buf;
+    }
+
+    for (i = 0; i < buf->nelts; i++) {
+        const char * s = ((char**)buf->elts)[i];
+        if(s) {
+            fprintf(out, "%s", s);
+	}
+    }
+    return buf;
+}
+
+/* Append a "%s" formatted string to a string buffer */
+struct Buf*
+buf_prints(struct Buf *buf, const char *fmt, const char *s)
+{
+    char *t;
+    size_t tsz;
+
+    t = malloc(tsz = strlen(fmt) + strlen(s) + 1);
+    snprintf(t, tsz, fmt, s);
+    buf = buf_strappend(buf, t);
+    free(t);
+    return buf;
+}
+
+/** Append a line directive to the string buffer.
+ * @param buf A string buffer.
+ * @param filename file name
+ * @param lineno line number
+ * @return buf
+ */
+struct Buf*
+buf_linedir(struct Buf *buf, const char* filename, int lineno)
+{
+    char   *t, *fmt = "#line %d \"%s\"\n";
+    size_t tsz;
+    
+    t = malloc(tsz = strlen(fmt) + strlen(filename) + (int)(1 + log10(lineno >= 0? lineno : -lineno)) + 1);
+    snprintf(t, tsz, fmt, lineno, filename);
+    buf = buf_strappend(buf, t);
+    free(t);
+    return buf;
+}
+
+
+/** Append the contents of @a src to @a dest.
+ * @param @a dest the destination buffer
+ * @param @a dest the source buffer
+ * @return @a dest
+ */
+struct Buf*
+buf_concat(struct Buf* dest, const struct Buf* src)
+{
+    buf_append(dest, src->elts, src->nelts);
+    return dest;
+}
+
+
+/* Appends n characters in str to buf. */
+struct Buf*
+buf_strnappend(struct Buf *buf, const char *str, int n)
+{
+    buf_append(buf, str, n + 1);
+
+    /* "undo" the '\0' character that buf_append() already copied. */
+    buf->nelts--;
+
+    return buf;
+}
+
+/* Appends characters in str to buf. */
+struct Buf*
+buf_strappend(struct Buf *buf, const char *str)
+{
+    return buf_strnappend(buf, str, strlen(str));
+}
+
+/* appends "#define str def\n" */
+struct Buf*
+buf_strdefine(struct Buf *buf, const char *str, const char *def)
+{
+    buf_strappend(buf, "#define ");
+    buf_strappend(buf, " ");
+    buf_strappend(buf, str);
+    buf_strappend(buf, " ");
+    buf_strappend(buf, def);
+    buf_strappend(buf, "\n");
+    return buf;
+}
+
+/** Pushes "m4_define( [[def]], [[val]])m4_dnl" to end of buffer.
+ * @param buf A buffer as a list of strings.
+ * @param def The m4 symbol to define.
+ * @param val The definition; may be NULL.
+ * @return buf
+ */
+struct Buf*
+buf_m4_define(struct Buf *buf, const char* def, const char* val)
+{
+    const char * fmt = "m4_define( [[%s]], [[%s]])m4_dnl\n";
+    char * str;
+    size_t strsz;
+
+    val = val?val:"";
+    str = (char*)malloc(strsz = strlen(fmt) + strlen(def) + strlen(val) + 2);
+
+    snprintf(str, strsz, fmt, def, val);
+    buf_append(buf, &str, 1);
+    return buf;
+}
+
+/** Pushes "m4_undefine([[def]])m4_dnl" to end of buffer.
+ * @param buf A buffer as a list of strings.
+ * @param def The m4 symbol to undefine.
+ * @return buf
+ */
+struct Buf*
+buf_m4_undefine(struct Buf *buf, const char* def)
+{
+    const char * fmt = "m4_undefine( [[%s]])m4_dnl\n";
+    char * str;
+    size_t strsz;
+
+    str = (char*)malloc(strsz = strlen(fmt) + strlen(def) + 2);
+
+    snprintf(str, strsz, fmt, def);
+    buf_append(buf, &str, 1);
+    return buf;
+}
+
+/* create buf with 0 elements, each of size elem_size. */
+void
+buf_init(struct Buf *buf, size_t elem_size)
+{
+    buf->elts = (void *) 0;
+    buf->nelts = 0;
+    buf->elt_size = elem_size;
+    buf->nmax = 0;
+}
+
+/* frees memory */
+void
+buf_destroy(struct Buf *buf)
+{
+    if (buf && buf->elts) {
+	free(buf->elts);
+    }
+    buf->elts = (void *) 0;
+}
+
+
+/* appends ptr[] to buf, grow if necessary.
+ * n_elem is number of elements in ptr[], NOT bytes.
+ * returns buf.
+ * We grow by mod(512) boundaries.
+ */
+struct Buf*
+buf_append(struct Buf *buf, const void *ptr, int n_elem)
+{
+    int n_alloc = 0;
+
+    if (!ptr || n_elem == 0) {
+	return buf;
+    }
+
+    /* May need to alloc more. */
+    if (n_elem + buf->nelts > buf->nmax) {
+	/* exact amount needed... */
+	n_alloc = (n_elem + buf->nelts) * buf->elt_size;
+
+	/* ...plus some extra */
+	if (((n_alloc * buf->elt_size) % 512) != 0 && buf->elt_size < 512) {
+	    n_alloc += (512 - ((n_alloc * buf->elt_size) % 512)) / buf->elt_size;
+	}
+	if (!buf->elts) {
+	    buf->elts = allocate_array(n_alloc, buf->elt_size);
+	} else {
+	    buf->elts = reallocate_array(buf->elts, n_alloc, buf->elt_size);
+	}
+	buf->nmax = n_alloc;
+    }
+    memcpy((char*)buf->elts + buf->nelts * buf->elt_size, ptr,
+	n_elem * buf->elt_size);
+
+    buf->nelts += n_elem;
+
+    return buf;
+}
+
+/* --- */
+/* input buffering support */
+
+/*!max:re2c*/
+
 /* Estimate of largest number of characters "in use" at once:
  * (<= YYMAXFILL input chars) + (<= YYMAXFILL backtracking chars) + '\0'
  */
@@ -24,7 +325,7 @@
 /* Size of scanner buffer. Should be big enough to comfortably hold
  * everything from backtracking marker to null marker.
  */
-static const int BUF_SIZE = (MAX_IN_USE > 128) ? MAX_IN_USE : 128;
+static const int BUF_SIZE = MAX_IN_USE;
 
 /* Copy up to n input characters to the end of scanner buffer.
  * If EOF is encountered before n characters are read, '\0'
@@ -51,7 +352,7 @@
     scanner->null = new;
 }
 
-/* Appends up to 2n characters of input to scanner buffer.
+/* Appends up to n characters of input to scanner buffer.
  *
  * Buffer contents are shifted if there is insufficient room
  * at the end of the buffer.
@@ -81,6 +382,19 @@
     bufferAppend(scanner, n);
 }
 
+static char*
+getTokenText(perplex_t scanner, struct Buf *localBuf)
+{
+    char term = '\0';
+    int tokenChars = scanner->cursor - scanner->tokenStart;
+
+    localBuf->nelts = 0;
+    buf_append(localBuf, scanner->tokenStart, tokenChars);
+    buf_append(localBuf, &term, sizeof(term));
+
+    return (char*)localBuf->elts;
+}
+
 static perplex_t
 newScanner()
 {
@@ -130,10 +444,14 @@
     free(scanner);
 }
 
-#define RETURN(id) return id
-#define YYGETCONDITION getCondition(scanner)
-#define YYFILL(n) bufferFill(scanner, n)
+#define YYGETCONDITION  getCondition(scanner)
+#define YYFILL(n)       bufferFill(scanner, n)
 
+#define UPDATE_START  scanner->tokenStart = scanner->cursor;
+#define yytext        getTokenText(scanner, _perplex_token_string)
+#define RETURN(id)    buf_destroy(_perplex_token_string); return id;
+#define CONTINUE      UPDATE_START; continue;
+
 int yylex(perplext_t scanner)
 {
 /*!re2c

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.