fix more lexical scanning and interpreter problems

[<ivt...@li...>]

Patch:    ivtools-040211-johnston-008
For:      ivtools-1.1.2
Author:   joh...@us...
Subject:  fix more lexical scanning and interpreter problems
Requires:

This is an intermediate patch to ivtools-1.1.2.  To apply, cd to the
top-level directory of the ivtools source tree (the directory with src
and config subdirs), and apply like this:

	patch -p0 <ThisFile

Summary of Changes:

- reinstate the skipping of "#" comments in shell scripts parsed by
the lexical scanner.  This can be avoided by passing a null byte to
lexscan(), or can be customized by passing a different character (the
new argument is called linecmt).

- fix various ComTerpServ functionality, smoothing the use of the new
fd filebuf constructor.

Index: Attribute/commodule.c
diff -c Attribute/commodule.c:1.1 Attribute/commodule.c:1.2
*** Attribute/commodule.c:1.1	Fri Jan  9 11:42:18 2004
--- src/Attribute/commodule.c	Wed Feb 11 12:03:28 2004
***************
*** 95,97 ****
--- 95,98 ----
       _token[0] = '\0';
       _linenum = 0;
   }
+
Index: Attribute/lexscan.c
diff -c Attribute/lexscan.c:1.1 Attribute/lexscan.c:1.2
*** Attribute/lexscan.c:1.1	Fri Jan  9 11:42:18 2004
--- src/Attribute/lexscan.c	Wed Feb 11 12:03:28 2004
***************
*** 63,69 ****
   {
       unsigned int toklen, tokstart;
       int status = lexscan (_inptr, _infunc, _eoffunc, _errfunc, NULL, 
NULL,
! 			  _begcmt, _endcmt, _buffer, _bufsiz, &_bufptr,
   			  _token, _toksiz, &toklen, &toktype, &tokstart, &_linenum);
       attr_value retval;
       switch (toktype) {
--- 63,69 ----
   {
       unsigned int toklen, tokstart;
       int status = lexscan (_inptr, _infunc, _eoffunc, _errfunc, NULL, 
NULL,
! 			  _begcmt, _endcmt, '#', _buffer, _bufsiz, &_bufptr,
   			  _token, _toksiz, &toklen, &toktype, &tokstart, &_linenum);
       attr_value retval;
       switch (toktype) {
***************
*** 87,93 ****
   {
       unsigned int toklen, tokstart;
       int status = lexscan (_inptr, _infunc, _eoffunc, _errfunc, NULL, 
NULL,
! 			  _begcmt, _endcmt, _buffer, _bufsiz, &_bufptr,
   			  _token, _toksiz, &toklen, &toktype, &tokstart, &_linenum);
       unsigned tok_buflen = _bufptr-tokstart;
       strncpy(_tokbuf, _buffer+tokstart, tok_buflen);
--- 87,93 ----
   {
       unsigned int toklen, tokstart;
       int status = lexscan (_inptr, _infunc, _eoffunc, _errfunc, NULL, 
NULL,
! 			  _begcmt, _endcmt, '#", _buffer, _bufsiz, &_bufptr,
   			  _token, _toksiz, &toklen, &toktype, &tokstart, &_linenum);
       unsigned tok_buflen = _bufptr-tokstart;
       strncpy(_tokbuf, _buffer+tokstart, tok_buflen);
***************
*** 119,121 ****
--- 119,122 ----

       return new AttributeValue(valtype, tokval);
   }
+
Index: ComTerp/comterp.c
diff -c ComTerp/comterp.c:1.2 ComTerp/comterp.c:1.3
*** ComTerp/comterp.c:1.2	Tue Feb 10 11:41:03 2004
--- src/ComTerp/comterp.c	Wed Feb 11 12:03:30 2004
***************
*** 139,144 ****
--- 139,145 ----
       _trace_mode = 0;
       _npause = 0;
       _stepflag = 0;
+
   }


***************
*** 884,890 ****
       fbuf.attach(fd);
     } else
       fbuf.attach(fileno(stdout));
! #elif __GNUC__==3 && __GNUC_MINOR__<1 || 1
     fileptr_filebuf fbuf(handler() && handler()->wrfptr()
   	       ? handler()->wrfptr() : stdout,
   	       ios_base::out);
--- 885,891 ----
       fbuf.attach(fd);
     } else
       fbuf.attach(fileno(stdout));
! #elif __GNUC__==3 && __GNUC_MINOR__<1
     fileptr_filebuf fbuf(handler() && handler()->wrfptr()
   	       ? handler()->wrfptr() : stdout,
   	       ios_base::out);
Index: ComTerp/comterpserv.c
diff -c ComTerp/comterpserv.c:1.1 ComTerp/comterpserv.c:1.2
*** ComTerp/comterpserv.c:1.1	Fri Jan  9 11:42:26 2004
--- src/ComTerp/comterpserv.c	Wed Feb 11 12:03:30 2004
***************
*** 152,163 ****
       fbuf.attach(fd);
       istream in (&fbuf);
       in.gets(&instr);
! #else
       char instr[BUFSIZ];
       FILE* ifptr = fdopen(fd, "r");
       fileptr_filebuf fbuf(ifptr, ios_base::in);
       istream in (&fbuf);
       in.get(instr, BUFSIZ, '\n');  // needs to be generalized with 
<vector.h>
   #endif
       server->_instat = in.good();

--- 152,168 ----
       fbuf.attach(fd);
       istream in (&fbuf);
       in.gets(&instr);
! #elif __GNUC__==3 && __GNUC_MINOR__<1
       char instr[BUFSIZ];
       FILE* ifptr = fdopen(fd, "r");
       fileptr_filebuf fbuf(ifptr, ios_base::in);
       istream in (&fbuf);
       in.get(instr, BUFSIZ, '\n');  // needs to be generalized with 
<vector.h>
+ #else
+     char instr[BUFSIZ];
+     fileptr_filebuf fbuf(fd, ios_base::in, false, 
static_cast<size_t>(BUFSIZ));
+     istream in (&fbuf);
+     in.get(instr, BUFSIZ, '\n');  // needs to be generalized with 
<vector.h>
   #endif
       server->_instat = in.good();

***************
*** 180,186 ****
       /* append a null byte */
       outstr[outpos] = '\0';

! #if __GNUC__>=3
       if (ifptr) fclose(ifptr);
   #endif

--- 185,191 ----
       /* append a null byte */
       outstr[outpos] = '\0';

! #if __GNUC__==3 && __GNUC_MINOR__<1
       if (ifptr) fclose(ifptr);
   #endif

***************
*** 197,212 ****
   #if __GNUC__<3
       filebuf fbuf;
       fbuf.attach(fd);
! #else
       FILE* ofptr = fdopen(fd, "w");
       fileptr_filebuf fbuf(ofptr, ios_base::out);
   #endif
       ostream out(&fbuf);
       for (; outpos < bufsize-1 && s[outpos]; outpos++)
   	out.put(s[outpos]);
       out.flush();
       outpos = 0;
! #if __GNUC__>=3
       if (ofptr) fclose(ofptr);
   #endif
       return 1;
--- 202,219 ----
   #if __GNUC__<3
       filebuf fbuf;
       fbuf.attach(fd);
! #elif __GNUC__==3 && __GNUC_MINOR__<1
       FILE* ofptr = fdopen(fd, "w");
       fileptr_filebuf fbuf(ofptr, ios_base::out);
+ #else
+     fileptr_filebuf fbuf(fd, ios_base::out, false, 
static_cast<size_t>(BUFSIZ));
   #endif
       ostream out(&fbuf);
       for (; outpos < bufsize-1 && s[outpos]; outpos++)
   	out.put(s[outpos]);
       out.flush();
       outpos = 0;
! #if __GNUC__==3 && __GNUC_MINOR__<1
       if (ofptr) fclose(ofptr);
   #endif
       return 1;
***************
*** 313,325 ****
   	        err_print( stderr, "comterp" );
   #if __GNUC__<3
   	        filebuf obuf(handler() ? handler()->get_handle() : 1);
! #else
                   FILE* ofptr = fdopen(handler() ? 
handler()->get_handle() : 1, "w");
   	        fileptr_filebuf obuf(ofptr, ios_base::out);
   #endif
   		ostream ostr(&obuf);
   		ostr.flush();
! #if __GNUC__>=3
                   if (ofptr) fclose(ofptr);
   #endif
   		status = -1;
--- 320,335 ----
   	        err_print( stderr, "comterp" );
   #if __GNUC__<3
   	        filebuf obuf(handler() ? handler()->get_handle() : 1);
! #elif __GNUC__==3 && __GNUC_MINOR__<1
                   FILE* ofptr = fdopen(handler() ? 
handler()->get_handle() : 1, "w");
   	        fileptr_filebuf obuf(ofptr, ios_base::out);
+ #else
+ 		fileptr_filebuf obuf(handler() ? (int)handler()->get_handle() : 1,
+ 				     ios_base::out, false, static_cast<size_t>(BUFSIZ));
   #endif
   		ostream ostr(&obuf);
   		ostr.flush();
! #if __GNUC__==3 && __GNUC_MINOR__<1
                   if (ofptr) fclose(ofptr);
   #endif
   		status = -1;
***************
*** 334,346 ****
   	  err_print( stderr, "comterp" );
   #if __GNUC__<3
   	  filebuf obuf(handler() ? handler()->get_handle() : 1);
! #else
             FILE* ofptr = fdopen(handler() ? handler()->get_handle() : 
1, "w");
   	  fileptr_filebuf obuf(ofptr, ios_base::out);
   #endif
   	  ostream ostr(&obuf);
   	  ostr.flush();
! #if __GNUC__>=3
             if (ofptr) fclose(ofptr);
   #endif
   	  status = -1;
--- 344,360 ----
   	  err_print( stderr, "comterp" );
   #if __GNUC__<3
   	  filebuf obuf(handler() ? handler()->get_handle() : 1);
! #elif __GNUC__==3 && __GNUC_MINOR__<1
             FILE* ofptr = fdopen(handler() ? handler()->get_handle() : 
1, "w");
   	  fileptr_filebuf obuf(ofptr, ios_base::out);
+ #else
+ 	  fileptr_filebuf obuf(handler() ? (int)handler()->get_handle() : 1,
+ 			       ios_base::out, false, static_cast<size_t>(BUFSIZ));
+
   #endif
   	  ostream ostr(&obuf);
   	  ostr.flush();
! #if __GNUC__==3 && __GNUC_MINOR__<1
             if (ofptr) fclose(ofptr);
   #endif
   	  status = -1;
Index: ComUtil/_lexscan.c
diff -c ComUtil/_lexscan.c:1.2 ComUtil/_lexscan.c:1.3
*** ComUtil/_lexscan.c:1.2	Tue Feb 10 11:41:01 2004
--- src/ComUtil/_lexscan.c	Wed Feb 11 12:03:25 2004
***************
*** 75,81 ****
   */

   int lexscan(void * infile,char * (*infunc)(),int (*eoffunc)(), int 
(*errfunc)(),
! 	    void * outfile,int (*outfunc)(),char * begcmt,char * endcmt,
               char * buffer,unsigned bufsiz,unsigned * bufptr,char * 
token,
   	    unsigned toksiz, unsigned * toklen,unsigned * toktype,
   	    unsigned * tokstart,unsigned * linenum )
--- 75,82 ----
   */

   int lexscan(void * infile,char * (*infunc)(),int (*eoffunc)(), int 
(*errfunc)(),
! 	    void * outfile,int (*outfunc)(),
! 	    char * begcmt,char * endcmt, char linecmt,
               char * buffer,unsigned bufsiz,unsigned * bufptr,char * 
token,
   	    unsigned toksiz, unsigned * toklen,unsigned * toktype,
   	    unsigned * tokstart,unsigned * linenum )
***************
*** 104,109 ****
--- 105,111 ----
   				  (typically `fputs`). */
   char *	        begcmt    ;/* I   String that begins comment. */
   char *          endcmt    ;/* I   String that ends comment. */
+ char            linecmt   ;/* I   Character to start comment line.  */
   char *          buffer    ;/* I   Buffer or file I/O.          */
   unsigned        bufsiz    ;/* I   Size of `buffer` in bytes.   */
   unsigned *      bufptr    ;/* O   Current location in `buffer`.*/
***************
*** 242,253 ****
   	     (*outfunc) ( "> ", outfile);
   	   _continuation_prompt = 0;
   	 }
! #if 0
! 	 while( (infunc_retval = (*infunc)( buffer, bufsiz, infile )) != 
NULL  &&
! 		buffer[0] == '#') {} /* skip all script comments */
! #else
! 	 infunc_retval = (*infunc)( buffer, bufsiz, infile );
! #endif
   	 if( infunc_retval == NULL ) {
   	    if( *toklen > 0 )
   	       goto token_return;
--- 244,254 ----
   	     (*outfunc) ( "> ", outfile);
   	   _continuation_prompt = 0;
   	 }
! 	 if (linecmt)
! 	   while( (infunc_retval = (*infunc)( buffer, bufsiz, infile )) != 
NULL &&
! 		  buffer[0] == linecmt) {} /* skip all script comments */
! 	 else
! 	   infunc_retval = (*infunc)( buffer, bufsiz, infile );
   	 if( infunc_retval == NULL ) {
   	    if( *toklen > 0 )
   	       goto token_return;
Index: ComUtil/_parser.c
diff -c ComUtil/_parser.c:1.1 ComUtil/_parser.c:1.2
*** ComUtil/_parser.c:1.1	Fri Jan  9 11:42:14 2004
--- src/ComUtil/_parser.c	Wed Feb 11 12:03:26 2004
***************
*** 40,45 ****
--- 40,46 ----

   int _continuation_prompt;
   int _continuation_prompt_disabled = 0;
+ int _skip_shell_comments = 0;
   infuncptr _oneshot_infunc;

   static int get_next_token(void *infile, char *(*infunc)(), int 
(*eoffunc)(),
Index: ComUtil/_scanner.c
diff -c ComUtil/_scanner.c:1.1 ComUtil/_scanner.c:1.2
*** ComUtil/_scanner.c:1.1	Fri Jan  9 11:42:14 2004
--- src/ComUtil/_scanner.c	Wed Feb 11 12:03:26 2004
***************
*** 148,154 ****
      /* Don't worry about status return, because token_type     */
      /* will be set to TOK_NONE                                 */
         status = lexscan( infile, infunc, eoffunc, errfunc, outfile, 
outfunc,
!                         "/*", "*/", buffer, bufsiz, bufptr,
                           token, toksiz, toklen, toktype,
                           tokstart, linenum );
         if( status != 0 )
--- 148,154 ----
      /* Don't worry about status return, because token_type     */
      /* will be set to TOK_NONE                                 */
         status = lexscan( infile, infunc, eoffunc, errfunc, outfile, 
outfunc,
!                         "/*", "*/", '#', buffer, bufsiz, bufptr,
                           token, toksiz, toklen, toktype,
                           tokstart, linenum );
         if( status != 0 )
Index: ComUtil/comutil.arg
diff -c ComUtil/comutil.arg:1.1 ComUtil/comutil.arg:1.2
*** ComUtil/comutil.arg:1.1	Fri Jan  9 11:42:15 2004
--- src/ComUtil/comutil.arg	Wed Feb 11 12:03:26 2004
***************
*** 56,69 ****
   int lexscan(void *infile,char *(*infunc)(char*,int,void*),
   	    int (*eoffunc)(void*),int (*errfunc)(void*),
   	    void *outfile,int (*outfunc)(const char*,void*),
! 	    char *begcmt,char *endcmt,
   	    char *buffer,unsigned bufsiz,unsigned *bufptr,char *token,
   	    unsigned toksiz,unsigned *toklen,unsigned *toktype,
   	    unsigned *tokstart,unsigned *linenum );
   #else
   int lexscan(void *infile,char *(*infunc)(),
   	    int (*eoffunc)(),int (*errfunc)(),
! 	    void *outfile,int (*outfunc)(),char *begcmt,char *endcmt,
   	    char *buffer,unsigned bufsiz,unsigned *bufptr,char *token,
   	    unsigned toksiz,unsigned *toklen,unsigned *toktype,
   	    unsigned *tokstart,unsigned *linenum );
--- 56,70 ----
   int lexscan(void *infile,char *(*infunc)(char*,int,void*),
   	    int (*eoffunc)(void*),int (*errfunc)(void*),
   	    void *outfile,int (*outfunc)(const char*,void*),
! 	    char *begcmt,char *endcmt,char linecmt,
   	    char *buffer,unsigned bufsiz,unsigned *bufptr,char *token,
   	    unsigned toksiz,unsigned *toklen,unsigned *toktype,
   	    unsigned *tokstart,unsigned *linenum );
   #else
   int lexscan(void *infile,char *(*infunc)(),
   	    int (*eoffunc)(),int (*errfunc)(),
! 	    void *outfile,int (*outfunc)(),
! 	    char *begcmt,char *endcmt,char linecmt,
   	    char *buffer,unsigned bufsiz,unsigned *bufptr,char *token,
   	    unsigned toksiz,unsigned *toklen,unsigned *toktype,
   	    unsigned *tokstart,unsigned *linenum );
Index: ComUtil/errfile.c
diff -c ComUtil/errfile.c:1.1 ComUtil/errfile.c:1.2
*** ComUtil/errfile.c:1.1	Fri Jan  9 11:42:15 2004
--- src/ComUtil/errfile.c	Wed Feb 11 12:03:26 2004
***************
*** 132,138 ****
      /* Don't worry about status return, because token_type     */
      /* will be set to TOK_NONE                                 */
         status = lexscan( errstream, fgets, ffeof, fferror,
! 			NULL, NULL, NULL, NULL,
   			InputBuffer, MAX_INPUT_LENGTH+2, &colnum,
   			FormatBuffer, MAX_FORMAT_LENGTH+1, &token_length,
   			&token_type, &token_start, &linenum );
--- 132,138 ----
      /* Don't worry about status return, because token_type     */
      /* will be set to TOK_NONE                                 */
         status = lexscan( errstream, fgets, ffeof, fferror,
! 			NULL, NULL, NULL, NULL, 0,
   			InputBuffer, MAX_INPUT_LENGTH+2, &colnum,
   			FormatBuffer, MAX_FORMAT_LENGTH+1, &token_length,
   			&token_type, &token_start, &linenum );
*** /dev/null	 Wed Feb 11 12:03:57 PST 2004
--- patches/ivtools-040211-johnston-008
*************** patches/ivtools-040211-johnston-008
*** 0 ****
--- 1 ----
+ ivtools-040211-johnston-008