|
From: <mb...@us...> - 2006-07-18 08:52:43
|
Revision: 316 Author: mbaas Date: 2006-07-18 01:52:35 -0700 (Tue, 18 Jul 2006) ViewCVS: http://svn.sourceforge.net/pygccxml/?rev=316&view=rev Log Message: ----------- Added *.pyc files to the ignore list Property Changed: ---------------- pyplusplus_dev/contrib/arrayinfo/ pyplusplus_dev/contrib/pypp_api/pypp_api/ Property changes on: pyplusplus_dev/contrib/arrayinfo ___________________________________________________________________ Name: svn:ignore + *.pyc Property changes on: pyplusplus_dev/contrib/pypp_api/pypp_api ___________________________________________________________________ Name: svn:ignore + *.pyc This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
|
From: <mb...@us...> - 2006-10-23 07:49:09
|
Revision: 681
http://svn.sourceforge.net/pygccxml/?rev=681&view=rev
Author: mbaas
Date: 2006-10-23 00:49:03 -0700 (Mon, 23 Oct 2006)
Log Message:
-----------
Committed the last couple of changes (handling pathes with blanks, eol-style, allowing the tokenizer to keep preprocessor directives).
Modified Paths:
--------------
pyplusplus_dev/contrib/arrayinfo/arrayinfo.py
pyplusplus_dev/contrib/arrayinfo/cpptokenize.py
pyplusplus_dev/contrib/arrayinfo/createarrayinfo.py
Modified: pyplusplus_dev/contrib/arrayinfo/arrayinfo.py
===================================================================
--- pyplusplus_dev/contrib/arrayinfo/arrayinfo.py 2006-10-23 07:38:41 UTC (rev 680)
+++ pyplusplus_dev/contrib/arrayinfo/arrayinfo.py 2006-10-23 07:49:03 UTC (rev 681)
@@ -1,71 +1,73 @@
-######################################################################
-# Array info
-#
-# Author: Matthias Baas (ba...@ir...)
-######################################################################
-
-# ArrayInfo
-class ArrayInfo:
- """Reads the information that was created by createarrayinfo.py.
-
- Usage:
-
- Create an instance of this class and either provide the file
- name in the constructor or call read() manually. After the info
- file has been read you can call arraySize() to retrieve the
- size of a particular argument.
- """
-
- def __init__(self, datfile=None):
- """Constructor.
-
- datfile is the name of the file that contains the array size
- information (as created by the createarrayinfo.py utility).
- If a name is provided is already read in (so there's no need
- to manually call read() anymore).
- """
-
- # Key: (filename, firstline, funcname)
- self.funcargs = {}
-
- if datfile!=None:
- self.read(datfile)
-
- # read
- def read(self, datfile):
- """Read an info file.
- """
-
- n = 0
- for s in file(datfile, "rt"):
- a = s.split(";")
- filename, firstline, lastline, classname, funcname, args = a
- firstline = int(firstline)
- lastline = int(lastline)
- args = eval(args)
- key = (filename, lastline, funcname)
- self.funcargs[key] = args
- n += 1
- print n,"entries read from",datfile
-
- # arraySize
- def arraySize(self, decl, idx):
- """Return the array size of an argument.
-
- decl is the pygccxml calldef declaration whose argument list
- should be inspected. idx is the index of the argument (0-based).
- The return value is None if the argument is no array or the
- array size is not known, otherwise the size of the array is
- returned.
- """
- filename = decl.location.file_name
- line = decl.location.line
- funcname = decl.name
- args = self.funcargs.get((filename, line, funcname))
- if args==None:
- return None
- if idx>=len(args):
- return None
- else:
- return args[idx]
-
+######################################################################
+# Array info
+#
+# Author: Matthias Baas (ba...@ir...)
+######################################################################
+
+import os.path
+
+# ArrayInfo
+class ArrayInfo:
+ """Reads the information that was created by createarrayinfo.py.
+
+ Usage:
+
+ Create an instance of this class and either provide the file
+ name in the constructor or call read() manually. After the info
+ file has been read you can call arraySize() to retrieve the
+ size of a particular argument.
+ """
+
+ def __init__(self, datfile=None):
+ """Constructor.
+
+ datfile is the name of the file that contains the array size
+ information (as created by the createarrayinfo.py utility).
+ If a name is provided is already read in (so there's no need
+ to manually call read() anymore).
+ """
+
+ # Key: (filename, firstline, funcname)
+ self.funcargs = {}
+
+ if datfile!=None:
+ self.read(datfile)
+
+ # read
+ def read(self, datfile):
+ """Read an info file.
+ """
+
+ n = 0
+ for s in file(datfile, "rt"):
+ a = s.split(";")
+ filename, firstline, lastline, classname, funcname, args = a
+ firstline = int(firstline)
+ lastline = int(lastline)
+ args = eval(args)
+ key = (filename, lastline, funcname)
+ self.funcargs[key] = args
+ n += 1
+ print n,"entries read from",datfile
+
+ # arraySize
+ def arraySize(self, decl, idx):
+ """Return the array size of an argument.
+
+ decl is the pygccxml calldef declaration whose argument list
+ should be inspected. idx is the index of the argument (0-based).
+ The return value is None if the argument is no array or the
+ array size is not known, otherwise the size of the array is
+ returned.
+ """
+ filename = os.path.normpath(decl.location.file_name)
+ line = decl.location.line
+ funcname = decl.name
+ args = self.funcargs.get((filename, line, funcname))
+ if args==None:
+ return None
+ if idx>=len(args):
+ return None
+ else:
+ return args[idx]
+
Modified: pyplusplus_dev/contrib/arrayinfo/cpptokenize.py
===================================================================
--- pyplusplus_dev/contrib/arrayinfo/cpptokenize.py 2006-10-23 07:38:41 UTC (rev 680)
+++ pyplusplus_dev/contrib/arrayinfo/cpptokenize.py 2006-10-23 07:49:03 UTC (rev 681)
@@ -1,119 +1,127 @@
-###########################################################################
-# C/C++ tokenizer
-# Copyright (C) 2006 Matthias Baas (ba...@ir...)
-###########################################################################
-
-"""C/C++ tokenizer module.
-
-This module provides the function tokenize() that just works like the
-equivalent Python function with the only difference that it scans
-C/C++ source code.
-"""
-
-import re
-
-WHITESPACE = 0
-NAME = 1
-NUMBER = 2
-STRING = 3
-NEWLINE = 4
-OPERATOR = 5
-CHARACTER = 6
-
-# tokenize
-def tokenize(readline, tokeater):
- """Reads a C/C++ input stream and creates tokens.
-
- The first parameter, readline, must be a callable object which
- provides the same interface as the readline() method of built-in
- file objects. Each call to the function should return one line of
- input as a string.
-
- The second parameter, tokeneater, must also be a callable object.
- It is called with six parameters: the token type, the token
- string, a tuple (srow, scol) specifying the row and column where
- the token begins in the source, a tuple (erow, ecol) giving the
- ending position of the token, the line on which the token was
- found and the filename of the current file.
-
- By default the filename argument is an empty string. It will only
- be the actual filename if you provide a preprocessed file stream
- as input (so you should first run cpp on any source code). The
- tokenizer actually expects preprocessed data as it doesn't handle
- comments.
- """
-
- regs = ( (WHITESPACE, re.compile(r"[ \t]+")),
- (NAME, re.compile(r"[A-Za-z_][A-Za-z_0-9]*")),
- (NUMBER, re.compile(r"[0-9]+(\.[0-9]+)?(E(\+|-)?[0-9]+)?")),
- (STRING, re.compile(r"\"[^\"]*\"|'[^\']*\'")),
- (OPERATOR, re.compile(r"->|::|\+\+|--|->\*|\.\*|<<|>>|<=|>=|==|!=|&&|\|\||\+=|-=|\*=|/=|%=|&=|\^=|\|=|<<=|>>=|\(|\)|\[|\]|\.|\+|-|!|~|\*|/|&|\^|%|<|>|\?|:|=")),
- (NEWLINE, re.compile(r"\n"))
- )
-
- linenr = 0
- filename = ""
- while 1:
- # Read next line
- line = readline()
- # No more lines? then finish
- if line=="":
- break
-
- linenr+=1
- # Base for starting column...
- scolbase = 0
-
- # Process preprocessor lines...
- if line[0]=="#":
- try:
- f = line.strip().split(" ")
- linenr = int(f[1])-1
- filename = f[2][1:-1]
- except:
- pass
- continue
-
- s = line
-
- # Create tokens...
- while s!="":
- unmatched=1
- # Check all regular expressions...
- for r in regs:
- m=r[1].match(s)
- # Does it match? then the token is found
- if m!=None:
- scol = m.start()
- ecol = m.end()
- tok = s[scol:ecol]
- s = s[ecol:]
- typ = r[0]
- tokeater(typ, tok, (linenr, scolbase+scol), (linenr, scolbase+ecol), line, filename)
- scolbase += ecol
- unmatched=0
- continue
-
- # No match? then report a single character...
- if unmatched:
- tok = s[0]
- tokeater(CHARACTER, tok, (linenr, scolbase), (linenr, scolbase+1), line, filename)
- s = s[1:]
- scolbase += 1
-
-
-def _tokeater(type, s, start, end, line, filename):
- """Test token eater."""
- if type==WHITESPACE or type==NEWLINE:
- return
- type_str = ["WHITESPACE", "NAME", "NUMBER", "STRING", "NEWLINE", "OPERATOR", "CHARACTER"]
-
- print "Token: %-11s %-20s %s %s %s"%(type_str[type],s, start,end,filename)
-
-######################################################################
-
-if __name__=="__main__":
-
- f=open("header.h")
- tokenize(f.readline, _tokeater)
-
\ No newline at end of file
+###########################################################################
+# C/C++ tokenizer
+# Copyright (C) 2006 Matthias Baas (ba...@ir...)
+###########################################################################
+
+"""C/C++ tokenizer module.
+
+This module provides the function tokenize() that just works like the
+equivalent Python function with the only difference that it scans
+C/C++ source code.
+"""
+
+import re, os.path
+
+WHITESPACE = 0
+NAME = 1
+NUMBER = 2
+STRING = 3
+NEWLINE = 4
+OPERATOR = 5
+CHARACTER = 6
+
+# tokenize
+def tokenize(readline, tokeater, preprocessed=True):
+ """Reads a C/C++ input stream and creates tokens.
+
+ The first parameter, readline, must be a callable object which
+ provides the same interface as the readline() method of built-in
+ file objects. Each call to the function should return one line of
+ input as a string.
+
+ The second parameter, tokeneater, must also be a callable object.
+ It is called with six parameters: the token type, the token
+ string, a tuple (srow, scol) specifying the row and column where
+ the token begins in the source, a tuple (erow, ecol) giving the
+ ending position of the token, the line on which the token was
+ found and the filename of the current file.
+
+ By default the filename argument is an empty string. It will only
+ be the actual filename if you provide a preprocessed file stream
+ as input (so you should first run cpp on any source code). The
+ tokenizer actually expects preprocessed data as it doesn't handle
+ comments. If preprocessed is False, any preprocessor directive is
+ also considered to be a token.
+ """
+
+ regs = ( (WHITESPACE, re.compile(r"[ \t]+")),
+ (NAME, re.compile(r"[A-Za-z_][A-Za-z_0-9]*")),
+ (NUMBER, re.compile(r"[0-9]+(\.[0-9]+)?(E(\+|-)?[0-9]+)?")),
+ (STRING, re.compile(r"\"[^\"]*\"|'[^\']*\'")),
+ (OPERATOR, re.compile(r"->|::|\+\+|--|->\*|\.\*|<<|>>|<=|>=|==|!=|&&|\|\||\+=|-=|\*=|/=|%=|&=|\^=|\|=|<<=|>>=|\(|\)|\[|\]|\.|\+|-|!|~|\*|/|&|\^|%|<|>|\?|:|=")),
+ (NEWLINE, re.compile(r"\n"))
+ )
+
+ refilename = re.compile(r'\"[^\"]*\"')
+
+ linenr = 0
+ filename = ""
+ while 1:
+ # Read next line
+ line = readline()
+ # No more lines? then finish
+ if line=="":
+ break
+
+ linenr+=1
+ # Base for starting column...
+ scolbase = 0
+
+ # Process preprocessor lines...
+ if preprocessed and line[0]=="#":
+ try:
+ line0 = line.strip()
+ f = line0.split(" ")
+ linenr = int(f[1])-1
+ m = refilename.search(line0)
+ if m==None:
+ filename = os.path.normpath(f[2][1:-1])
+ else:
+ filename = line0[m.start()+1:m.end()-1]
+ except:
+ pass
+ continue
+
+ s = line
+
+ # Create tokens...
+ while s!="":
+ unmatched=1
+ # Check all regular expressions...
+ for r in regs:
+ m=r[1].match(s)
+ # Does it match? then the token is found
+ if m!=None:
+ scol = m.start()
+ ecol = m.end()
+ tok = s[scol:ecol]
+ s = s[ecol:]
+ typ = r[0]
+ tokeater(typ, tok, (linenr, scolbase+scol), (linenr, scolbase+ecol), line, filename)
+ scolbase += ecol
+ unmatched=0
+ continue
+
+ # No match? then report a single character...
+ if unmatched:
+ tok = s[0]
+ tokeater(CHARACTER, tok, (linenr, scolbase), (linenr, scolbase+1), line, filename)
+ s = s[1:]
+ scolbase += 1
+
+
+def _tokeater(type, s, start, end, line, filename):
+ """Test token eater."""
+ if type==WHITESPACE or type==NEWLINE:
+ return
+ type_str = ["WHITESPACE", "NAME", "NUMBER", "STRING", "NEWLINE", "OPERATOR", "CHARACTER"]
+
+ print "Token: %-11s %-20s %s %s %s"%(type_str[type],s, start,end,filename)
+
+######################################################################
+
+if __name__=="__main__":
+
+ f=open("header.h")
+ tokenize(f.readline, _tokeater)
+
Modified: pyplusplus_dev/contrib/arrayinfo/createarrayinfo.py
===================================================================
--- pyplusplus_dev/contrib/arrayinfo/createarrayinfo.py 2006-10-23 07:38:41 UTC (rev 680)
+++ pyplusplus_dev/contrib/arrayinfo/createarrayinfo.py 2006-10-23 07:49:03 UTC (rev 681)
@@ -1,169 +1,169 @@
-#!/usr/bin/env python
-######################################################################
-# This tool parses header files inside a directory and stores info
-# about the array size of function arguments into a file for later
-# retrieval.
-#
-# usage: createarrayinfo.py [options] <headerpath>
-#
-# options:
-# -h, --help show this help message and exit
-# -o FILENAME, --output=FILENAME
-# Output file name (default: stdout)
-# -a ARGSTRING, --cppargs=ARGSTRING
-# Additional C preproceesor arguments
-#
-# Author: Matthias Baas (ba...@ir...)
-######################################################################
-
-import sys, os, os.path, glob, optparse
-from cpptokenize import *
-
-# Parser
-class Parser:
- """Parser class.
-
- This class contains the token eater method that processes the tokens
- generated by cpptokenize.
- Whenever a function signature is parsed a line is written to the output.
- """
-
- def __init__(self, headerpath, output=sys.stdout):
- """Constructor.
- """
-
- self.headerpath = os.path.normpath(headerpath)
- self.output = output
-
- # Buffer for the last NAME token (which might be a function name)
- self.lastname = None
- # The current state
- self.state = "Outside"
-
- self.classname = None
- self.funcname = None
- self.args = None
- self.arraysize = None
- self.no_arg = False
- self.firstline = None
-
- def tokeater(self, type, s, start, end, line, filename):
- """Token eater."""
- if type==WHITESPACE or type==NEWLINE:
- return
-
- method = getattr(self, "state%s"%self.state)
- method(type, s, start, end, line, filename)
-
- # The state methods. They are called by the token eater and must take
- # the same arguments than the token eater.
-
- def stateOutside(self, type, s, start, end, line, filename):
- if type==NAME and s=="class":
- self.state = "ClassName"
- if type==NAME:
- self.firstline = start[0]
- self.lastname = s
- elif self.lastname=="operator":
- self.lastname += s
- elif type==OPERATOR and s=="(":
- self.funcname = self.lastname
- self.args = []
- self.arraysize = None
- self.no_arg = True
- self.state = "Args"
-
- def stateClassName(self, type, s, start, end, line, filename):
- if s.upper()==s:
- return
- self.classname = s
- self.state = "Outside"
-
- def stateArgs(self, type, s, start, end, line, filename):
- if s==")":
- if not self.no_arg:
- self.args.append(self.arraysize)
- self.state = "End"
- elif s==",":
- self.args.append(self.arraysize)
- self.arraysize = None
- elif s=="[":
- self.state = "ArgsSize"
- self.no_arg = False
-
- def stateArgsSize(self, type, s, start, end, line, filename):
- if s=="]":
- self.state = "Args"
- else:
- self.arraysize = int(s)
- self.state = "Args"
-
- def stateEnd(self, type, s, start, end, line, filename):
- if s==";":
- if os.path.normpath(os.path.dirname(filename))==self.headerpath:
- self.onFuncComplete(self.classname, self.funcname, self.args, self.firstline, end[0], filename)
- self.state = "Outside"
-
-
- def onFuncComplete(self, classname, funcname, args, firstline, lastline, filename):
- """Callback that is called when one function is completely processed.
- """
- print >>self.output, "%s;%d;%d;%s;%s;%s"%(filename, firstline, lastline, classname, funcname, args)
-
-
-# parseHeader
-def parseHeader(filename, cpp="cpp", cppargs="", output=sys.stdout):
- """Parse a header file.
-
- filename is the header file name and cppargs is a string with
- additional arguments for the invocation of the preprocessor 'cpp'.
- output is the output stream.
- """
- # Run the file through the preprocessor...
- filename = os.path.abspath(filename)
- print >>sys.stderr, "Parsing",filename
- cmd = "%s %s %s >_tmp.h"%(cpp, cppargs, filename)
- print >>sys.stderr, cmd
- os.system(cmd)
-
- # Parse the preprocessed file...
- parser = Parser(os.path.dirname(filename), output)
- tokenize(file("_tmp.h").readline, parser.tokeater)
-
-######################################################################
-
-# Preprocessor
-cpp = "cpp"
-# Preprocessor arguments
-cppargs = ""
-# Output stream
-output = sys.stdout
-
-usage = "usage: %prog [options] <headerpath>"
-op = optparse.OptionParser(usage)
-op.add_option("-o", "--output", metavar="FILENAME",
- help="Output file name")
-op.add_option("-a", "--cppargs", metavar="ARGSTRING", default="",
- help="Additional C preproceesor arguments")
-
-options, args = op.parse_args()
-
-if len(args)==0:
- op.print_help()
- sys.exit(1)
-
-if options.output!=None:
- print >>sys.stderr, "Output file: %s"%options.output
- output = file(options.output, "wt")
-
-headerpath = args[0]
-headernames = os.path.join(headerpath, "*.h")
-print >>sys.stderr, "Input files: %s"%headernames
-
-cppargs = options.cppargs
-print >>sys.stderr, "Preprocessor args: %s"%cppargs
-
-headers = glob.glob(headernames)
-print >>sys.stderr, "%d header files found"%len(headers)
-for header in headers:
- parseHeader(header, cpp, cppargs, output)
\ No newline at end of file
+#!/usr/bin/env python
+######################################################################
+# This tool parses header files inside a directory and stores info
+# about the array size of function arguments into a file for later
+# retrieval.
+#
+# usage: createarrayinfo.py [options] <headerpath>
+#
+# options:
+# -h, --help show this help message and exit
+# -o FILENAME, --output=FILENAME
+# Output file name (default: stdout)
+# -a ARGSTRING, --cppargs=ARGSTRING
+# Additional C preproceesor arguments
+#
+# Author: Matthias Baas (ba...@ir...)
+######################################################################
+
+import sys, os, os.path, glob, optparse
+from cpptokenize import *
+
+# Parser
+class Parser:
+ """Parser class.
+
+ This class contains the token eater method that processes the tokens
+ generated by cpptokenize.
+ Whenever a function signature is parsed a line is written to the output.
+ """
+
+ def __init__(self, headerpath, output=sys.stdout):
+ """Constructor.
+ """
+
+ self.headerpath = os.path.normpath(headerpath)
+ self.output = output
+
+ # Buffer for the last NAME token (which might be a function name)
+ self.lastname = None
+ # The current state
+ self.state = "Outside"
+
+ self.classname = None
+ self.funcname = None
+ self.args = None
+ self.arraysize = None
+ self.no_arg = False
+ self.firstline = None
+
+ def tokeater(self, type, s, start, end, line, filename):
+ """Token eater."""
+ if type==WHITESPACE or type==NEWLINE:
+ return
+
+ method = getattr(self, "state%s"%self.state)
+ method(type, s, start, end, line, filename)
+
+ # The state methods. They are called by the token eater and must take
+ # the same arguments than the token eater.
+
+ def stateOutside(self, type, s, start, end, line, filename):
+ if type==NAME and s=="class":
+ self.state = "ClassName"
+ if type==NAME:
+ self.firstline = start[0]
+ self.lastname = s
+ elif self.lastname=="operator":
+ self.lastname += s
+ elif type==OPERATOR and s=="(":
+ self.funcname = self.lastname
+ self.args = []
+ self.arraysize = None
+ self.no_arg = True
+ self.state = "Args"
+
+ def stateClassName(self, type, s, start, end, line, filename):
+ if s.upper()==s:
+ return
+ self.classname = s
+ self.state = "Outside"
+
+ def stateArgs(self, type, s, start, end, line, filename):
+ if s==")":
+ if not self.no_arg:
+ self.args.append(self.arraysize)
+ self.state = "End"
+ elif s==",":
+ self.args.append(self.arraysize)
+ self.arraysize = None
+ elif s=="[":
+ self.state = "ArgsSize"
+ self.no_arg = False
+
+ def stateArgsSize(self, type, s, start, end, line, filename):
+ if s=="]":
+ self.state = "Args"
+ else:
+ self.arraysize = int(s)
+ self.state = "Args"
+
+ def stateEnd(self, type, s, start, end, line, filename):
+ if s==";":
+ if os.path.normpath(os.path.dirname(filename))==self.headerpath:
+ self.onFuncComplete(self.classname, self.funcname, self.args, self.firstline, end[0], filename)
+ self.state = "Outside"
+
+
+ def onFuncComplete(self, classname, funcname, args, firstline, lastline, filename):
+ """Callback that is called when one function is completely processed.
+ """
+ print >>self.output, "%s;%d;%d;%s;%s;%s"%(filename, firstline, lastline, classname, funcname, args)
+
+
+# parseHeader
+def parseHeader(filename, cpp="cpp", cppargs="", output=sys.stdout):
+ """Parse a header file.
+
+ filename is the header file name and cppargs is a string with
+ additional arguments for the invocation of the preprocessor 'cpp'.
+ output is the output stream.
+ """
+ # Run the file through the preprocessor...
+ filename = os.path.abspath(filename)
+ print >>sys.stderr, "Parsing",filename
+ cmd = '%s %s "%s" >_tmp.h'%(cpp, cppargs, filename.replace(" ", "\\ "))
+ print >>sys.stderr, cmd
+ os.system(cmd)
+
+ # Parse the preprocessed file...
+ parser = Parser(os.path.dirname(filename), output)
+ tokenize(file("_tmp.h").readline, parser.tokeater)
+
+######################################################################
+
+# Preprocessor
+cpp = "cpp"
+# Preprocessor arguments
+cppargs = ""
+# Output stream
+output = sys.stdout
+
+usage = "usage: %prog [options] <headerpath>"
+op = optparse.OptionParser(usage)
+op.add_option("-o", "--output", metavar="FILENAME",
+ help="Output file name")
+op.add_option("-a", "--cppargs", metavar="ARGSTRING", default="",
+ help="Additional C preproceesor arguments")
+
+options, args = op.parse_args()
+
+if len(args)==0:
+ op.print_help()
+ sys.exit(1)
+
+if options.output!=None:
+ print >>sys.stderr, "Output file: %s"%options.output
+ output = file(options.output, "wt")
+
+headerpath = args[0]
+headernames = os.path.join(headerpath, "*.h")
+print >>sys.stderr, "Input files: %s"%headernames
+
+cppargs = options.cppargs
+print >>sys.stderr, "Preprocessor args: %s"%cppargs
+
+headers = glob.glob(headernames)
+print >>sys.stderr, "%d header files found"%len(headers)
+for header in headers:
+ parseHeader(header, cpp, cppargs, output)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
