[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/parsing pathutils.py,NONE,1.1 init.py,NON

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/pydev/org.python.pydev/PySrc/ThirdParty/brm/bike/parsing
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9579/PySrc/ThirdParty/brm/bike/parsing

Added Files:
	pathutils.py __init__.py constants.py load.py newstuff.py 
	visitor.py fastparser.py fastparserast.py parserutils.py 
	utils.py 
Log Message:
Code completion improvements.
Starting refactoring integration with bicycle repair man.

--- NEW FILE: fastparserast.py ---
from __future__ import generators
from parserutils import generateLogicalLines, maskStringsAndComments, maskStringsAndRemoveComments
import re
import os
import compiler
from bike.transformer.save import resetOutputQueue

TABWIDTH = 4

classNameRE = re.compile("^\s*class\s+(\w+)")
fnNameRE = re.compile("^\s*def\s+(\w+)")

_root = None

def getRoot():
    global _root
    if _root is None:
        resetRoot()
    return _root 

def resetRoot(root = None):
    global _root
    _root = root or Root()
    _root.unittestmode = False
    resetOutputQueue()

def getModule(filename_path):
    from bike.parsing.load import CantLocateSourceNodeException, getSourceNode
    try:
        sourcenode = getSourceNode(filename_path)
        return sourcenode.fastparseroot
    except CantLocateSourceNodeException:
        return None

def getPackage(directory_path):
    from bike.parsing.pathutils import getRootDirectory
    rootdir = getRootDirectory(directory_path)
    if rootdir == directory_path:
        return getRoot()
    else:
        return Package(directory_path,
                       os.path.basename(directory_path))

class Root:
    def __init__(self, pythonpath = None):
        # singleton hack to allow functions in query package to appear
        # 'stateless'
        resetRoot(self)

        # this is to get round a python optimisation which reuses an
        # empty list as a default arg. unfortunately the client of
        # this method may fill that list, so it's not empty
        if not pythonpath:
            pythonpath = []
        self.pythonpath = pythonpath

    def __repr__(self):
        return "Root()"
        #return "Root(%s)"%(self.getChildNodes())

    # dummy method
    def getChild(self,name):
        return None

class Package:
    def __init__(self, path, name):
        self.path = path
        self.name = name

    def getChild(self,name):
        from bike.parsing.newstuff import getModule
        return getModule(os.path.join(self.path,name+".py"))

    def __repr__(self):
        return "Package(%s,%s)"%(self.path, self.name)

# used so that linenum can be an attribute
class Line(str):
    pass

class StructuralNode:
    def __init__(self, filename, srclines, modulesrc):
        self.childNodes = []
        self.filename = filename
        self._parent = None
        self._modulesrc = modulesrc
        self._srclines = srclines
        self._maskedLines = None

    def addChild(self, node):
        self.childNodes.append(node)
        node.setParent(self)

    def setParent(self, parent):
        self._parent = parent

    def getParent(self):
        return self._parent

    def getChildNodes(self):
        return self.childNodes

    def getChild(self,name):
        matches = [c for c in self.getChildNodes() if c.name == name]
        if matches != []:
            return matches[0]

    def getLogicalLine(self,physicalLineno):
        return generateLogicalLines(self._srclines[physicalLineno-1:]).next()

    # badly named: actually returns line numbers of import statements
    def getImportLineNumbers(self):
        try:
            return self.importlines
        except AttributeError:
            return[]

    def getLinesNotIncludingThoseBelongingToChildScopes(self):
        srclines = self.getMaskedModuleLines()
        lines = []
        lineno = self.getStartLine()
        for child in self.getChildNodes():
            lines+=srclines[lineno-1: child.getStartLine()-1]
            lineno = child.getEndLine()
        lines+=srclines[lineno-1: self.getEndLine()-1]
        return lines

    def generateLinesNotIncludingThoseBelongingToChildScopes(self):
        srclines = self.getMaskedModuleLines()
        lines = []
        lineno = self.getStartLine()
        for child in self.getChildNodes():
            for line in srclines[lineno-1: child.getStartLine()-1]:
                yield self.attachLinenum(line,lineno)
                lineno +=1
            lineno = child.getEndLine()
        for line in srclines[lineno-1: self.getEndLine()-1]:
            yield self.attachLinenum(line,lineno)
            lineno +=1

    def generateLinesWithLineNumbers(self,startline=1):
        srclines = self.getMaskedModuleLines()
        for lineno in range(startline,len(srclines)+1):
            yield self.attachLinenum(srclines[lineno-1],lineno)

    def attachLinenum(self,line,lineno):
        line = Line(line)
        line.linenum = lineno
        return line

    def getMaskedModuleLines(self):
        from bike.parsing.load import Cache
        try:
            maskedlines = Cache.instance.maskedlinescache[self.filename]
        except:
            # make sure src is actually masked
            # (could just have keywords masked)
            maskedsrc = maskStringsAndComments(self._modulesrc)
            maskedlines = maskedsrc.splitlines(1)
            Cache.instance.maskedlinescache[self.filename] = maskedlines
        return maskedlines

class Module(StructuralNode):
    def __init__(self, filename, name, srclines, maskedsrc):
        StructuralNode.__init__(self, filename, srclines, maskedsrc)
        self.name = name
        self.indent = -TABWIDTH
        self.flattenedNodes = []
        self.module = self

    def getMaskedLines(self):
        return self.getMaskedModuleLines()

    def getFlattenedListOfChildNodes(self):
        return self.flattenedNodes

    def getStartLine(self):
        return 1

    def getEndLine(self):
        return len(self.getMaskedModuleLines())+1

    def getSourceNode(self):
        return self.sourcenode

    def setSourceNode(self, sourcenode):
        self.sourcenode = sourcenode

    def matchesCompilerNode(self,node):
        return isinstance(node,compiler.ast.Module) and \
               node.name == self.name

    def getParent(self):
        if self._parent is not None:
            return self._parent
        else:
            from newstuff import getPackage
            return getPackage(os.path.dirname(self.filename))

    def __str__(self):
        return "bike:Module:"+self.filename

indentRE = re.compile("^(\s*)\S")
class Node:
    # module = the module node
    # linenum = starting line number
    def __init__(self, name, module, linenum, indent):
        self.name = name
        self.module = module
        self.linenum = linenum
        self.endline = None
        self.indent = indent

    def getMaskedLines(self):
        return self.getMaskedModuleLines()[self.getStartLine()-1:self.getEndLine()-1]

    def getStartLine(self):
        return self.linenum

    def getEndLine(self):
        if self.endline is None:
            physicallines = self.getMaskedModuleLines()
            lineno = self.linenum
            logicallines = generateLogicalLines(physicallines[lineno-1:])

            # skip the first line, because it's the declaration
            line = logicallines.next()
            lineno+=line.count("\n")

            # scan to the end of the fn
            for line in logicallines:
                #print lineno,":",line,
                match = indentRE.match(line)
                if match and match.end()-1 <= self.indent:
                    break
                lineno+=line.count("\n")
            self.endline = lineno
        return self.endline

    # linenum starts at 0
    def getLine(self, linenum):
        return self._srclines[(self.getStartLine()-1) + linenum]

baseClassesRE = re.compile("class\s+[^(]+\(([^)]+)\):")

class Class(StructuralNode, Node):
    def __init__(self, name, filename, module, linenum, indent, srclines, maskedmodulesrc):
        StructuralNode.__init__(self, filename, srclines, maskedmodulesrc)
        Node.__init__(self, name, module, linenum, indent)
        self.type = "Class"

    def getBaseClassNames(self):
        #line = self.getLine(0)
        line = self.getLogicalLine(self.getStartLine())
        match = baseClassesRE.search(line)
        if match:
            return [s.strip()for s in match.group(1).split(",")]
        else:
            return []

    def getColumnOfName(self):
        match = classNameRE.match(self.getLine(0))
        return match.start(1)

    def __repr__(self):
        return "<bike:Class:%s>" % self.name

    def __str__(self):
        return "bike:Class:"+self.filename+":"+\
               str(self.getStartLine())+":"+self.name

    def matchesCompilerNode(self,node):
        return isinstance(node,compiler.ast.Class) and \
               node.name == self.name

    def __eq__(self,other):
        return isinstance(other,Class) and \
               self.filename == other.filename and \
               self.getStartLine() == other.getStartLine()

# describes an instance of a class
class Instance:
    def __init__(self, type):
        assert type is not None
        self._type = type

    def getType(self):
        return self._type

    def __str__(self):
        return "Instance(%s)"%(self.getType())

class Function(StructuralNode, Node):
    def __init__(self, name, filename, module, linenum, indent,
                 srclines, maskedsrc):
        StructuralNode.__init__(self, filename, srclines, maskedsrc)
        Node.__init__(self, name, module, linenum, indent)
        self.type = "Function"

    def getColumnOfName(self):
        match = fnNameRE.match(self.getLine(0))
        return match.start(1)

    def __repr__(self):
        return "<bike:Function:%s>" % self.name

    def __str__(self):
        return "bike:Function:"+self.filename+":"+\
               str(self.getStartLine())+":"+self.name

    def matchesCompilerNode(self,node):
        return isinstance(node,compiler.ast.Function) and \
               node.name == self.name

--- NEW FILE: fastparser.py ---
#!/usr/bin/env python
from bike.parsing.fastparserast import *
from bike.parsing.parserutils import *
from parser import ParserError
#import exceptions

indentRE = re.compile("^\s*(\w+)")

# returns a tree of objects representing nested classes and functions
# in the source
def fastparser(src,modulename="",filename=""):
    try:
        return fastparser_impl(src,modulename,filename)
    except RuntimeError, ex:   # if recursive call exceeds maximum depth
        if str(ex) == "maximum recursion limit exceeded":
            raise ParserError,"maximum recursion depth exceeded when fast-parsing src "+filename
        else:
            raise

def fastparser_impl(src,modulename,filename):
    lines = src.splitlines(1)
    maskedSrc = maskPythonKeywordsInStringsAndComments(src)
    maskedLines = maskedSrc.splitlines(1)
    root = Module(filename,modulename,lines,maskedSrc)
    parentnode = root
    lineno = 0
    for line in maskedLines:
        lineno+=1
        #print "line",lineno,":",line
        m = indentRE.match(line)
        if m:
            indent = m.start(1)
            tokenstr = m.group(1)
            if tokenstr == "import" or tokenstr == "from":
                while indent <= parentnode.indent:   # root indent is -TABWIDTH
                    parentnode = parentnode.getParent()
                try:
                    parentnode.importlines.append(lineno)
                except AttributeError:
                    parentnode.importlines = [lineno]
            elif tokenstr == "class":
                m2 = classNameRE.match(line)
                if m2:
                    n = Class(m2.group(1), filename, root, lineno, indent, lines, maskedSrc)
                    root.flattenedNodes.append(n)

                    while indent <= parentnode.indent:
                        parentnode = parentnode.getParent()
                    parentnode.addChild(n)
                    parentnode = n

            elif tokenstr == "def":
                m2 = fnNameRE.match(line)
                if m2:
                    n = Function(m2.group(1), filename, root, lineno, indent, lines, maskedSrc)
                    root.flattenedNodes.append(n)

                    while indent <= parentnode.indent:
                        parentnode = parentnode.getParent()
                    parentnode.addChild(n)
                    parentnode = n

            elif indent <= parentnode.indent and \
                     tokenstr in ['if','for','while','try']:
                parentnode = parentnode.getParent()
                while indent <= parentnode.indent:
                    parentnode = parentnode.getParent()

    return root

--- NEW FILE: constants.py ---
messages={"PARSING":"Parsing","ADDTYPEINFO":"Deducing Type Information"}

MAXCALLDEPTH = 10

--- NEW FILE: load.py ---
from bike.globals import *
import os
from bike.parsing.fastparser import fastparser

class Cache:
    def __init__(self):
        self.reset()
    def reset(self):
        self.srcnodecache = {}
        self.typecache = {}
        self.maskedlinescache = {}

    instance = None

Cache.instance = Cache()

class CantLocateSourceNodeException(Exception): pass

def getSourceNode(filename_path):
    #print "getSourceNode:",filename_path
    sourcenode = None

    try:
        sourcenode = Cache.instance.srcnodecache[filename_path]
    except KeyError:
        pass

    if sourcenode is None:
        from bike.parsing.newstuff import translateFnameToModuleName
        sourcenode = SourceFile.createFromFile(filename_path,
                              translateFnameToModuleName(filename_path))
    if sourcenode is None:
        raise CantLocateSourceNodeException(filename_path)

    Cache.instance.srcnodecache[filename_path]=sourcenode
    return sourcenode

class SourceFile:

    def createFromString(filename, modulename, src):
        return SourceFile(filename,modulename,src)
    createFromString = staticmethod(createFromString)

    def createFromFile(filename,modulename):
        try:
            f = file(filename)
            src = f.read()
            f.close()
        except IOError:
            return None
        else:
            return SourceFile(filename,modulename,src)

    createFromFile = staticmethod(createFromFile)

    def __init__(self, filename, modulename, src):

        if os.path.isabs(filename):
            self.filename = filename
        else:
            self.filename = os.path.abspath(filename)
        self.modulename = modulename

        self.resetWithSource(src)

    def resetWithSource(self, source):
        # fastparser ast
        self.fastparseroot = fastparser(source,self.modulename,self.filename)
        self.fastparseroot.setSourceNode(self)
        self._lines = source.splitlines(1)
        self.sourcenode = self

    def __repr__(self):
        return "Source(%s,%s)"%('source', self.filename)

    def getChildNodes(self):
        return self.fastparseroot.getChildNodes()        

    def getSource(self):
        return  "".join(self.getLines())

    def getLine(self,linenum):
        return self.getLines()[linenum-1]

    # TODO: rename me!
    def getFlattenedListOfFastParserASTNodes(self):
        return self.fastparseroot.getFlattenedListOfChildNodes()

    def getLines(self):
        return self._lines

--- NEW FILE: parserutils.py ---
from __future__ import generators
import re

escapedQuotesRE = re.compile(r"(\\\\|\\\"|\\\')")

# changess \" \' and \\ into ** so that text searches
# for " and ' won't hit escaped ones
def maskEscapedQuotes(src):
    return escapedQuotesRE.sub("**", src)

stringsAndCommentsRE =  \
      re.compile("(\"\"\".*?\"\"\"|'''.*?'''|\"[^\"]*\"|\'[^\']*\'|#.*?\n)", re.DOTALL)

import string
#transtable = string.maketrans('classdefifforwhiletry', "*********************")

# performs a transformation on all of the comments and strings so that
# text searches for python keywords won't accidently find a keyword in
# a string or comment
def maskPythonKeywordsInStringsAndComments(src):
    src = escapedQuotesRE.sub("**", src)
    allstrings = stringsAndCommentsRE.split(src)
    # every odd element is a string or comment
    for i in xrange(1, len(allstrings), 2):
        allstrings[i] = allstrings[i].upper()
        #allstrings[i] = allstrings[i].translate(transtable)
    return "".join(allstrings)

allchars = string.maketrans("", "")
allcharsExceptNewline = allchars[: allchars.index('\n')]+allchars[allchars.index('\n')+1:]
allcharsExceptNewlineTranstable = string.maketrans(allcharsExceptNewline, '*'*len(allcharsExceptNewline))

# replaces all chars in a string or a comment with * (except newlines).
# this ensures that text searches don't mistake comments for keywords, and that all
# matches are in the same line/comment as the original
def maskStringsAndComments(src):
    src = escapedQuotesRE.sub("**", src)
    allstrings = stringsAndCommentsRE.split(src)
    # every odd element is a string or comment
    for i in xrange(1, len(allstrings), 2):
        if allstrings[i].startswith("'''")or allstrings[i].startswith('"""'):
            allstrings[i] = allstrings[i][:3]+ \
                           allstrings[i][3:-3].translate(allcharsExceptNewlineTranstable)+ \
                           allstrings[i][-3:]
        else:
            allstrings[i] = allstrings[i][0]+ \
                           allstrings[i][1:-1].translate(allcharsExceptNewlineTranstable)+ \
                           allstrings[i][-1]

    return "".join(allstrings)

# replaces all chars in a string or a comment with * (except newlines).
# this ensures that text searches don't mistake comments for keywords, and that all
# matches are in the same line/comment as the original
def maskStringsAndRemoveComments(src):
    src = escapedQuotesRE.sub("**", src)
    allstrings = stringsAndCommentsRE.split(src)
    # every odd element is a string or comment
    for i in xrange(1, len(allstrings), 2):
        if allstrings[i].startswith("'''")or allstrings[i].startswith('"""'):
            allstrings[i] = allstrings[i][:3]+ \
                           allstrings[i][3:-3].translate(allcharsExceptNewlineTranstable)+ \
                           allstrings[i][-3:]
        elif allstrings[i].startswith("#"):
            allstrings[i] = '\n'
        else:
            allstrings[i] = allstrings[i][0]+ \
                           allstrings[i][1:-1].translate(allcharsExceptNewlineTranstable)+ \
                           allstrings[i][-1]
    return "".join(allstrings)

implicitContinuationChars = (('(', ')'), ('[', ']'), ('{', '}'))
emptyHangingBraces = [0,0,0,0,0]
linecontinueRE = re.compile(r"\\\s*(#.*)?$")
multiLineStringsRE =  \
      re.compile("(^.*?\"\"\".*?\"\"\".*?$|^.*?'''.*?'''.*?$)", re.DOTALL)

#def splitLogicalLines(src):
#    src = multiLineStringsRE.split(src)

# splits the string into logical lines.  This requires the comments to
# be removed, and strings masked (see other fns in this module)
def splitLogicalLines(src):
    physicallines = src.splitlines(1)
    return [x for x in generateLogicalLines(physicallines)]

class UnbalancedBracesException: pass

# splits the string into logical lines.  This requires the strings
# masked (see other fns in this module)
# Physical Lines *Must* start on a non-continued non-in-a-comment line
# (although detects unbalanced braces)
def generateLogicalLines(physicallines):
    tmp = []
    hangingBraces = list(emptyHangingBraces)
    hangingComments = 0
    for line in physicallines:
        # update hanging braces
        for i in range(len(implicitContinuationChars)):
            contchar = implicitContinuationChars[i]
            numHanging = hangingBraces[i]
            hangingBraces[i] = numHanging+line.count(contchar[0]) - \
                               line.count(contchar[1])

        hangingComments ^= line.count('"""') % 2
        hangingComments ^= line.count("'''") % 2

        if hangingBraces[0] < 0 or \
           hangingBraces[1] < 0 or \
           hangingBraces[2] < 0:
            raise UnbalancedBracesException()

        if linecontinueRE.search(line):
            tmp.append(line)
        elif hangingBraces != emptyHangingBraces:
            tmp.append(line)
        elif hangingComments:
            tmp.append(line)
        else:
            tmp.append(line)
            yield "".join(tmp)
            tmp = []

# see above but yields (line,linenum)
#   needs physicallines to have linenum attribute
#   TODO: refactor with previous function
def generateLogicalLinesAndLineNumbers(physicallines):
    tmp = []
    hangingBraces = list(emptyHangingBraces)
    hangingComments = 0
    linenum = None
    for line in physicallines:
        if tmp == []:
            linenum = line.linenum

        # update hanging braces
        for i in range(len(implicitContinuationChars)):
            contchar = implicitContinuationChars[i]
            numHanging = hangingBraces[i]
            hangingBraces[i] = numHanging+line.count(contchar[0]) - \
                               line.count(contchar[1])

        hangingComments ^= line.count('"""') % 2
        hangingComments ^= line.count("'''") % 2

        if linecontinueRE.search(line):
            tmp.append(line)
        elif hangingBraces != emptyHangingBraces:
            tmp.append(line)
        elif hangingComments:
            tmp.append(line)
        else:
            tmp.append(line)
            yield "".join(tmp),linenum
            tmp = []

# takes a line of code, and decorates it with noops so that it can be
# parsed by the python compiler.
# e.g.  "if foo:"  -> "if foo: pass"
# returns the line, and the adjustment made to the column pos of the first char
# line must have strings and comments masked
#
# N.B. it only inserts keywords whitespace and 0's
notSpaceRE = re.compile("\s*(\S)")
commentRE = re.compile("#.*$")

def makeLineParseable(line):
    return makeLineParseableWhenCommentsRemoved(commentRE.sub("",line))

def makeLineParseableWhenCommentsRemoved(line):
    line = line.strip()
    if ":" in line:
        if line.endswith(":"):
            line += " pass"
        if line.startswith("try"):
            line += "\nexcept: pass"
        elif line.startswith("except") or line.startswith("finally"):
            line = "try: pass\n" + line
            return line
        elif line.startswith("else") or line.startswith("elif"):
            line = "if 0: pass\n" + line
            return line
    elif line.startswith("yield"):
        return ("return"+line[5:])
    return line

--- NEW FILE: utils.py ---
# get the first element of a fully qualified python path
#(e.g. _car('a.b.c.d') = 'a')
def fqn_car(fqn):
    try:
        return fqn[:fqn.index(".")]
    except ValueError:   # i.e. no dots in fqn
        return fqn

# get the other elements of a fully qualified python path
#(e.g. _cdr('a.b.c.d') = 'b.c.d')
def fqn_cdr(fqn):
    try:
        return fqn[fqn.index(".")+1:]
    except ValueError:   # i.e. no dots in fqn
        return ""

# reverse of above _rcar("a.b.c.d") = "d"
def fqn_rcar(fqn):
    try:
        return fqn[fqn.rindex(".")+1:]
    except ValueError:   # i.e. no dots in fqn
        return fqn

# reverse of above _rcdr("a.b.c.d") = "a.b.c"
def fqn_rcdr(fqn):
    try:
        return fqn[:fqn.rindex(".")]
    except ValueError:   # i.e. no dots in fqn
        return ""

--- NEW FILE: pathutils.py ---

# A some of this code is take from Pythius -
# Copyright (GPL) 2001 Jurgen Hermann <jh...@we...>

from bike.globals import *
import os

def containsAny(str, set):
    """ Check whether 'str' contains ANY of the chars in 'set'
    """
    return 1 in [c in str for c in set]

def getPathOfModuleOrPackage(dotted_name, pathlist = None):
    """ Get the filesystem path for a module or a package.

        Return the file system path to a file for a module,
        and to a directory for a package. Return None if
        the name is not found, or is a builtin or extension module.
    """
    from bike.parsing.newstuff import getPythonPath
    if pathlist is None:
        pathlist = getPythonPath()

    import imp

    # split off top-most name
    parts = dotted_name.split('.', 1)

    if len(parts) > 1:
        # we have a dotted path, import top-level package
        try:
            file, pathname, description = imp.find_module(parts[0], pathlist)
            if file: file.close()
        except ImportError:
            return None

        # check if it's indeed a package
        if description[2] == imp.PKG_DIRECTORY:
            # recursively handle the remaining name parts
            pathname = getPathOfModuleOrPackage(parts[1], [pathname])
        else:
            pathname = None
    else:
        # plain name
        try:
            file, pathname, description = imp.find_module(dotted_name, pathlist)
            if file: file.close()
            if description[2]not in[imp.PY_SOURCE, imp.PKG_DIRECTORY]:
                pathname = None
        except ImportError:
            pathname = None

    return pathname

def getFilesForName(name):
    """ Get a list of module files for a filename, a module or package name,
        or a directory.
    """
    import imp

    if not os.path.exists(name):
        # check for glob chars
        if containsAny(name, "*?[]"):
            import glob
            files = glob.glob(name)
            list = []
            for file in files:
                list.extend(getFilesForName(file))
            return list

        # try to find module or package
        name = getPathOfModuleOrPackage(name)
        if not name:
            return[]

    if os.path.isdir(name):
        # find all python files in directory
        list = []
        os.path.walk(name, _visit_pyfiles, list)
        return list
    elif os.path.exists(name) and not name.startswith("."):
        # a single file
        return [name]

    return []

def _visit_pyfiles(list, dirname, names):
    """ Helper for getFilesForName().
    """
    # get extension for python source files
    if not globals().has_key('_py_ext'):
        import imp
        global _py_ext
        _py_ext = [triple[0]for triple in imp.get_suffixes()if triple[2] == imp.PY_SOURCE][0]

    # don't recurse into CVS or Subversion directories
    if 'CVS'in names:
        names.remove('CVS')
    if '.svn'in names:
        names.remove('.svn')

    names_copy = [] + names
    for n in names_copy:
        if os.path.isdir(os.path.join(dirname, n))and \
           not os.path.exists(os.path.join(dirname, n, "__init__.py")):
            names.remove(n)

    # add all *.py files to list
    list.extend(
    [os.path.join(dirname, file)
    for file in names
    if os.path.splitext(file)[1] == _py_ext and not file.startswith(".")])

# returns the directory which holds the first package of the package
# hierarchy under which 'filename' belongs
def getRootDirectory(filename):
    if os.path.isdir(filename):
        dir = filename
    else:
        dir = os.path.dirname(filename)
    while dir != "" and \
          os.path.exists(os.path.join(dir, "__init__.py")):
        dir = os.path.dirname(dir)
    return dir

# Returns the root package directoryname of the package hierarchy
# under which 'filename' belongs
def getPackageBaseDirectory(filename):
    if os.path.isdir(filename):
        dir = filename
    else:
        dir = os.path.dirname(filename)

    if not os.path.exists(os.path.join(dir, "__init__.py")):
        # parent dir is not a package
        return dir

    while dir != "" and \
          os.path.exists(os.path.join(os.path.dirname(dir), "__init__.py")):
        dir = os.path.dirname(dir)
    return dir

def filenameToModulePath(fname):
    directoriesPreceedingRoot = getRootDirectory(fname)
    import os
    # strip off directories preceeding root package directory
    if directoriesPreceedingRoot != "":
        mpath = fname.replace(directoriesPreceedingRoot, "")
    else:
        mpath = fname

    if(mpath[0] == os.path.normpath("/")):
        mpath = mpath[1:]
    mpath, ext = os.path.splitext(mpath)
    mpath = mpath.replace(os.path.normpath("/"), ".")
    return mpath

--- NEW FILE: newstuff.py ---
from __future__ import generators 
# Holding module for scaffolding needed to transition parsing package
# into stateless design
import os
import re
from bike.parsing.pathutils import getRootDirectory, getPackageBaseDirectory, \
     filenameToModulePath, getPathOfModuleOrPackage, getFilesForName
from bike.parsing.fastparserast import Module, Package, getRoot, getPackage, getModule 
import sys
from bike.parsing.load import getSourceNode, CantLocateSourceNodeException

def translateFnameToModuleName(filename_path):
    return filenameToModulePath(filename_path)

# scope is the scope to search from
def getModuleOrPackageUsingFQN(fqn, dirpath=None):
    pythonpath = getPythonPath()
    #print "getModuleOrPackageUsingFQN",pythonpath,fqn
    if dirpath is not None:
        assert os.path.isdir(dirpath)
        pythonpath = [dirpath] + pythonpath
    filename = getPathOfModuleOrPackage(fqn,pythonpath)
    #print "getModuleOrPackageUsingFQN - filename",filename
    if filename is not None:
        if os.path.isdir(filename):
            return getPackage(filename)
        else:
            return getModule(filename)
    else:
        return None

def getPythonPath():
    return getRoot().pythonpath

def generateModuleFilenamesInPythonPath(contextFilename):
    files = []
    rootdir = getRootDirectory(contextFilename)
    if rootdir in getPythonPath():
        # just search the pythonpath
        for path in getPythonPath():
            for file in getFilesForName(path):
                if file not in files:   # check for duplicates
                    files.append(file)
                    yield file
    else:
        # search the package hierarchy containing contextFilename
        # in addition to pythonpath
        basedir = getPackageBaseDirectory(contextFilename)
        for path in [basedir] + getPythonPath():
            for file in getFilesForName(path):
                if file not in files:   # check for duplicates
                    files.append(file)
                    yield file

        # and search the files immediately above the package hierarchy
        for file in getFilesForName(os.path.join(rootdir,"*.py")):
            if file not in files:   # check for duplicates
                files.append(file)
                yield file

def generateModuleFilenamesInPackage(filenameInPackage):
    basedir = getPackageBaseDirectory(filenameInPackage)
    for file in getFilesForName(basedir):
        yield file

# search all sourcenodes globally from the perspective of file 'contextFilename'
def getSourceNodesContainingRegex(regexstr,contextFilename):
    regex = re.compile(regexstr)
    for fname in generateModuleFilenamesInPythonPath(contextFilename):
        try:
            f = file(fname)
            src = f.read()
        finally:
            f.close()
        if regex.search(src) is not None:
            yield getSourceNode(fname)

fromRegex = re.compile("^\s*from\s+(\w+)\s+import")
importregex = re.compile("^\s*import\s+(\w+)")

# fileInPackage is the filename of a file in the package hierarchy
# generates file and directory paths
def generatePackageDependencies(fileInPackage):
    rejectPackagePaths = [getPackageBaseDirectory(fileInPackage)]
    for fname in generateModuleFilenamesInPackage(fileInPackage):

        try:
            f = file(fname)
            src = f.read()
        finally:
            f.close()

        packagepath = None

        for line in src.splitlines():
            match = fromRegex.search(line) or importregex.search(line)
            if match is not None:
                modulepath = match.group(1)
                packagename = modulepath.split('.')[0]
                packagepath = getPathOfModuleOrPackage(packagename,
                                                       getPythonPath())
            if packagepath is not None and \
                   packagepath not in rejectPackagePaths:
                rejectPackagePaths.append(packagepath) # avoid duplicates
                yield packagepath

--- NEW FILE: visitor.py ---
from __future__ import generators

class TreeWalker(object):
    VERBOSE = 0

    def __init__(self):
        self.node = None
        self._cache = {}

    def default(self, node, *args):
        for child in node.getChildNodes():
            self.dispatch(child, *args)

    def dispatch(self, node, *args):
        self.node = node
        klass = node.__class__
        meth = self._cache.get(klass, None)
        if meth is None:
            className = klass.__name__
            meth = getattr(self.visitor, 'visit' + className, self.default)
            self._cache[klass] = meth
        return meth(node, *args)

    def preorder(self, tree, visitor, *args):
        """Do preorder walk of tree using visitor"""
        self.visitor = visitor
        visitor.visit = self.dispatch
        visitor.visitChildren = self.default
        return self.dispatch(tree, *args)

class GeneratingTreeWalker(TreeWalker):

    def default(self, node, *args):
        for child in node.getChildNodes():
            for i in self.dispatch(child, *args):
                yield i

def walk(tree, visitor):
    walker = TreeWalker()
    walker.preorder(tree, visitor)
    return walker.visitor

def walkAndGenerate(tree,visitor):
    walker = GeneratingTreeWalker()
    return walker.preorder(tree, visitor)

--- NEW FILE: __init__.py ---
#from addtypeinfo import addtypeinfo
#from load import load
#from brmtransformer import parse

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/parsing pathutils.py,NONE,1.1 __init__.py,NON

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/parsing pathutils.py,NONE,1.1 __init__.py,NONE,1.1 constants.py,NONE,1.1 load.py,NONE,1.1 newstuff.py,NONE,1.1 visitor.py,NONE,1.1 fastparser.py,NONE,1.1 fastparserast.py,NONE,1.1 parserutils.py,NONE,1.1 utils.py,NONE,1.1

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/parsing pathutils.py,NONE,1.1 init.py,NON

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/parsing pathutils.py,NONE,1.1 init.py,NONE,1.1 constants.py,NONE,1.1 load.py,NONE,1.1 newstuff.py,NONE,1.1 visitor.py,NONE,1.1 fastparser.py,NONE,1.1 fastparserast.py,NONE,1.1 parserutils.py,NONE,1.1 utils.py,NONE,1.1