[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/query getTypeOf.py,NONE,1.1 __init__.py,NONE,1.1 common.py,NONE,1.1 findReferences.py,NONE,1.1 relationships.py,NONE,1.1 getAllRelatedClasses.py,NONE,1.1 getReferencesToModule.py,NONE,1.1 findDefinition.py,NONE,1.1

[Fabio Z. <fa...@us...>]

Update of /cvsroot/pydev/org.python.pydev/PySrc/ThirdParty/brm/bike/query
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9579/PySrc/ThirdParty/brm/bike/query

Added Files:
	getTypeOf.py __init__.py common.py findReferences.py 
	relationships.py getAllRelatedClasses.py 
	getReferencesToModule.py findDefinition.py 
Log Message:
Code completion improvements.
Starting refactoring integration with bicycle repair man.


--- NEW FILE: getReferencesToModule.py ---
from __future__ import generators
from bike.query.common import Match, globalScanForMatches, getScopeForLine, MatchFinder
from getTypeOf import getTypeOf, getTypeOfExpr
import compiler
import re

def getReferencesToModule(root, fqn):
    
    modulename = fqn.split(".")[-1]
    moduleobj = getTypeOf(root, fqn)
    moduleRefFinder = ModuleRefFinder(moduleobj)
    
    for ref in globalScanForMatches(moduleRefFinder, modulename):
        yield ref

        
class ModuleRefFinder(MatchFinder):
    def __init__(self, targetmodule):
        self.targetmodule = targetmodule

    def visitName(self, node):
        if node.name == self.targetmodule.name:
            if getTypeOfExpr(self.scope, node) == self.targetmodule:
                self.appendMatch(node.name)
        self.popWordsUpTo(node.name)

    def visitImport(self, node):
        for name, alias in node.names:
            if name.split(".")[-1] == self.targetmodule.name:
                if getTypeOf(self.scope, name) == self.targetmodule:
                    self.appendMatch(self.targetmodule.name)
            for nameelem in name.split("."):
                self.popWordsUpTo(nameelem)
            if alias is not None:
                self.popWordsUpTo(alias)

    def visitGetattr(self, node):
        for c in node.getChildNodes():
            self.visit(c)
        if node.attrname == self.targetmodule.name:
            if getTypeOfExpr(self.scope, node) == self.targetmodule:
                self.appendMatch(self.targetmodule.name)
        self.popWordsUpTo(node.attrname)

    def visitFrom(self, node):
        for elem in node.modname.split("."):
            if elem == self.targetmodule.name:
                getTypeOf(self.scope, elem) == self.targetmodule
                self.appendMatch(self.targetmodule.name)
            self.popWordsUpTo(elem)
            
        for name, alias in node.names:
            if name == self.targetmodule.name:
                if alias and \
                   getTypeOf(self.scope, alias) == self.targetmodule:
                    self.appendMatch(self.targetmodule.name)
                elif getTypeOf(self.scope, name) == self.targetmodule:
                    self.appendMatch(self.targetmodule.name)
            if name != "*":
                self.popWordsUpTo(name)
            if alias is not None:
                self.popWordsUpTo(alias)

--- NEW FILE: findDefinition.py ---
from __future__ import generators
from bike.query.common import Match, MatchFinder, \
     getScopeForLine, indexToCoordinates, \
     translateSourceCoordsIntoASTNode, scanScopeForMatches, \
     isAMethod, convertNodeToMatchObject, walkLinesContainingStrings
from bike.parsing.parserutils import generateLogicalLines,\
     generateLogicalLinesAndLineNumbers, \
     splitLogicalLines, makeLineParseable
import compiler
from compiler.ast import Getattr, Name, AssName, AssAttr
from bike.parsing.fastparserast import getRoot, Package, Class, \
                                       Module, Function, Instance
import re
from bike.query.getTypeOf import getTypeOfExpr, UnfoundType, \
     isWordInLine, resolveImportedModuleOrPackage
from bike.parsing import visitor
from bike.parsing.visitor import walkAndGenerate

from bike.parsing.parserutils import makeLineParseable,splitLogicalLines
from bike.parsing.newstuff import getSourceNodesContainingRegex
from bike.parsing.load import getSourceNode
from bike import log


class CantFindDefinitionException:
    pass


def findAllPossibleDefinitionsByCoords(filepath,lineno,col):
    
    #try:
    node = translateSourceCoordsIntoASTNode(filepath,lineno,col)
    #except:
    #    import traceback
    #    traceback.print_exc()

    if node is None:
        raise "selected node type not supported"
    scope = getScopeForLine(getSourceNode(filepath),lineno)
    match = findDefinitionFromASTNode(scope,node)
    if match is not None:
        yield match
    if isinstance(node,Getattr) and (match is None or match.confidence != 100):
        root = getRoot()
        name = node.attrname
        for match in scanPythonPathForMatchingMethodNames(name,filepath):
            yield match
    print >>log.progress,"done"


def findDefinitionFromASTNode(scope,node):
    assert node is not None
    if isinstance(node,Name) or isinstance(node,AssName):
        while 1:
            # try scope children
            childscope = scope.getChild(node.name)
            if childscope is not None:
                return convertNodeToMatchObject(childscope,100)

            if isinstance(scope,Package):
                scope = scope.getChild("__init__")

            # try arguments and assignments
            match = scanScopeAST(scope,node.name,
                                 AssignmentAndFnArgsSearcher(node.name))
            if match is not None:
                return match
            
            # try imports
            match = searchImportedModulesForDefinition(scope,node)
            if match is not None:
                return match


            if not isinstance(scope,Module):
                # try parent scope
                scope = scope.getParent()
            else:
                break
        assert isinstance(scope,Module)

    elif isinstance(node,Getattr) or isinstance(node,AssAttr):
        exprtype = getTypeOfExpr(scope,node.expr)
        if not (exprtype is None or isinstance(exprtype,UnfoundType)):
            if isinstance(exprtype,Instance):
                exprtype = exprtype.getType()
                match = findDefinitionOfAttributeFromASTNode(exprtype,
                                                         node.attrname)
            else:
                match = findDefinitionFromASTNode(exprtype,
                                                  Name(node.attrname))
            if match is not None:
                return match

    elif isinstance(node,compiler.ast.Function) or \
             isinstance(node,compiler.ast.Class):
        if isAMethod(scope,node): 
            match = findDefinitionOfAttributeFromASTNode(scope,
                                                        node.name)
        else:
            match = findDefinitionFromASTNode(scope,Name(node.name))
        if match is not None:
            return match


    type = getTypeOfExpr(scope,node)
    if type is not None and (not isinstance(type,UnfoundType)) and \
                             (not isinstance(type,Instance)):
        return  convertNodeToMatchObject(type,100)
    else:
        return None


def findDefinitionOfAttributeFromASTNode(type,name):
    assert isinstance(type,Class)
    attrfinder = AttrbuteDefnFinder([type],name)

    # first scan the method names:
    for child in type.getChildNodes():
        if child.name == name:
            return convertNodeToMatchObject(child,100)
    # then scan the method source for attribues
    for child in type.getChildNodes():
        if isinstance(child,Function):
            try:
                return scanScopeForMatches(child.module.getSourceNode(),
                                        child, attrfinder,
                                        name).next()
            except StopIteration:
                continue


class AttrbuteDefnFinder(MatchFinder):
    def __init__(self,targetClasses,targetAttribute):
        self.targetClasses = targetClasses
        self.targetAttributeName = targetAttribute

    def visitAssAttr(self, node):
        for c in node.getChildNodes():
            self.visit(c)

        if node.attrname == self.targetAttributeName:
            exprtype = getTypeOfExpr(self.scope,node.expr)
            if isinstance(exprtype,Instance) and \
               exprtype.getType() in self.targetClasses:
                self.appendMatch(self.targetAttributeName)
            #else:
            #    self.appendMatch(self.targetAttributeName,50)
        self.popWordsUpTo(node.attrname)


    

def searchImportedModulesForDefinition(scope,node):
    lines = scope.module.getSourceNode().getLines()
    for lineno in scope.getImportLineNumbers():
        logicalline = getLogicalLine(lines,lineno)
        logicalline = makeLineParseable(logicalline)
        ast = compiler.parse(logicalline)
        class ImportVisitor:
            def __init__(self,node):
                self.target = node
                self.match = None
                assert isinstance(self.target,Name), \
                       "Getattr not supported"
                
            def visitFrom(self, node):
                module = resolveImportedModuleOrPackage(scope,node.modname)
                if module is None: # couldn't find module
                    return 
                
                if node.names[0][0] == '*': # e.g. from foo import *
                    match = findDefinitionFromASTNode(module,self.target)
                    if match is not None:
                        self.match = match
                    return
                    
                for name, alias in node.names:
                    if alias is None and name == self.target.name:
                        match = findDefinitionFromASTNode(module,self.target)
                        if match is not None:
                            self.match = match
                        return


        match = visitor.walk(ast, ImportVisitor(node)).match
        if match:
            return match
    # loop


def getLogicalLine(lines,lineno):
    return generateLogicalLines(lines[lineno-1:]).next()

class AssignmentAndFnArgsSearcher(MatchFinder):
    def __init__(self,name):
        self.targetname = name
        self.match = None

    def visitAssName(self, node):
        if node.name == self.targetname:
            idx = self.getNextIndexOfWord(self.targetname)
            self.match = idx
            return 

    def visitFunction(self, node):
        self.popWordsUpTo(node.name)
        for arg, default in self.zipArgs(node.argnames, node.defaults):
            if arg == self.targetname:
                idx = self.getNextIndexOfWord(self.targetname)
                self.match = idx
                return
            self.popWordsUpTo(arg)
            if default is not None:
                self.visit(default)
        self.visit(node.code)

    def getMatch(self):
        return self.match



# scans for lines containing keyword, and then runs the visitor over
# the parsed AST for that line
def scanScopeAST(scope,keyword,matchfinder):
    lines = scope.generateLinesNotIncludingThoseBelongingToChildScopes()
    match = None
    for line,linenum in generateLogicalLinesAndLineNumbers(lines):
        if isWordInLine(keyword, line):
            doctoredline = makeLineParseable(line)
            ast = compiler.parse(doctoredline)
            matchfinder.reset(line)
            match = visitor.walk(ast,matchfinder).getMatch()
            if match is not None:
                column,yoffset = indexToCoordinates(line,match)
                m = createMatch(scope,linenum + yoffset,column)
                return m
    return None

def createMatch(scope,lineno,x):
    m = Match()
    m.sourcenode = scope.module.getSourceNode()
    m.filename = m.sourcenode.filename
    m.lineno = lineno
    m.colno = x
    m.confidence = 100
    return m

# scan for methods globally (from perspective of 'perspectiveFilename')
def scanPythonPathForMatchingMethodNames(name, contextFilename):
    class MethodFinder:
        def __init__(self,srcnode):
            self.matches = []
            self.srcnode = srcnode
        def visitFunction(self,node):
            node = getScopeForLine(self.srcnode, self.lineno)
            if isinstance(node.getParent(),Class):
                if node.name == name:
                    self.matches.append(convertNodeToMatchObject(node,50))

    for srcnode in getSourceNodesContainingRegex(name,contextFilename):
        m = MethodFinder(srcnode)
        walkLinesContainingStrings(srcnode.fastparseroot,m,[name])
        for match in m.matches:
            yield match


def getIndexOfWord(line,targetword):
    words = re.split("(\w+)", line)
    idx = 0
    for word in words:
        if word == targetword:
            break
        idx += len(word)
    return idx



--- NEW FILE: getAllRelatedClasses.py ---


"""

def getAllRelatedClasses(root,classfqn):
    classobj = getTypeOf(root,classfqn)
    rootClasses = _getRootClasses(classobj)
    #print rootClasses
    relatedClasses = [] + rootClasses
    for rootClass in rootClasses:
        relatedClasses += _getAllSubClasses(rootClass,root)
    return relatedClasses

def _getRootClasses(klass):
    if klass is None:  # i.e. dont have base class in our ast
        return None
    if klass.getBaseClassNames() == []:  # i.e. is a root class
        return[klass]
    else:
        rootclasses = []
        for base in klass.getBaseClassNames():
            baseclass = getTypeOf(klass,base)
            rootclass = _getRootClasses(baseclass)
            if rootclass is None:  # base class not in our ast
                rootclass = [klass]
            rootclasses+=rootclass
        return rootclasses


def _getAllSubClasses(baseclass, root, subclasses = []):
    class ClassVisitor:
        def visitSource(self,node):
            self.visit(node.fastparseroot)
            
        def visitClass(self, node):
            for basename in node.getBaseClassNames():
                if basename.find(baseclass.name) != -1 and \
                       getTypeOf(node,basename) == baseclass:
                    subclasses.append(node)
                    _getAllSubClasses(node,root,subclasses)
            for child in node.getChildNodes():
                self.visit(child)
                
    walk(root, ClassVisitor())
    return subclasses

"""

--- NEW FILE: getTypeOf.py ---
# getTypeOf(scope,fqn) and getTypeOfExpr(scope,ast)

from bike.parsing.fastparserast import Class, Function, Module, Root, getRoot, Package, Instance, getModule
from bike.parsing.parserutils import generateLogicalLines, makeLineParseable,splitLogicalLines, makeLineParseable
from bike.parsing import visitor
from bike import log
from bike.parsing.newstuff import getModuleOrPackageUsingFQN
from bike.parsing.load import Cache
import os
import re
import compiler

# used if an assignment exists, but cant find type
# e.g. a = SomeFunctionNotLoaded()
# (as opposed to 'None' if cant find an assignment)
class UnfoundType: pass


getTypeOfStack = []

# name is the fqn of the reference, scope is the scope ast object from
# which the question is being asked.
# returns an fastparser-ast object representing the type
# or None if type not found
def getTypeOf(scope, fqn):
    if isinstance(scope, Root):
        assert False, "Can't use getTypeOf to resolve from Root. Use getModuleOrPackageUsingFQN instead"


    #print "getTypeOf:"+fqn+" -- "+str(scope)
    #print 
    #print str(getTypeOfStack)
    #print 
    if (fqn,scope) in getTypeOfStack:   # loop protection
        return None

    # this is crap!
    hashcode = str(scope)+fqn

    try:
        getTypeOfStack.append((fqn,scope))

        try:
            type = Cache.instance.typecache[hashcode]
        except KeyError:
            type = getTypeOf_impl(scope, fqn)
            Cache.instance.typecache[hashcode] = type
        return type
    finally:
        del getTypeOfStack[-1]
        


def getTypeOf_impl(scope, fqn):
    #print "getTypeOf_impl",scope,fqn
    if fqn == "None":
        return None

    if "."in fqn:
        rcdr = ".".join(fqn.split(".")[:-1])
        rcar = fqn.split(".")[-1]
        newscope = getTypeOf(scope,rcdr)
        if newscope is not None:
            return getTypeOf(newscope, rcar)
        else: 
            #print "couldnt find "+rcdr+" in "+str(scope)
            pass

    assert scope is not None
    #assert not ("." in fqn) 
    
    if isinstance(scope,UnfoundType):
        return UnfoundType()
    
    if isinstance(scope, Package):
        #assert 0,scope
        return handlePackageScope(scope, fqn)
    elif isinstance(scope,Instance):
        return handleClassInstanceAttribute(scope, fqn)
    else:
        return handleModuleClassOrFunctionScope(scope,fqn)



def handleModuleClassOrFunctionScope(scope,name):
    if name == "self" and isinstance(scope,Function) and \
           isinstance(scope.getParent(),Class):
        return Instance(scope.getParent())
    
    matches = [c for c in scope.getChildNodes()if c.name == name]
    if matches != []:
        return matches[0]
    
    type = scanScopeSourceForType(scope, name)
    if type != None:
        return type
    
    #print "name = ",name,"scope = ",scope
    type = getImportedType(scope, name)   # try imported types
    #print "type=",type
    if type != None:
        return type
    parentScope = scope.getParent()
    while isinstance(parentScope,Class):
        # don't search class scope, since this is not accessible except
        # through self   (is this true?)
        parentScope = parentScope.getParent()

    if not (isinstance(parentScope,Package) or isinstance(parentScope,Root)):
        return getTypeOf(parentScope, name)


def handleClassInstanceAttribute(instance, attrname):
    theClass = instance.getType()

    # search methods and inner classes
    match = theClass.getChild(attrname)
    if match:
        return match

    #search methods for assignments with self.foo getattrs
    for child in theClass.getChildNodes():
        if not isinstance(child,Function):
            continue
        res = scanScopeAST(child,attrname,
                          SelfAttributeAssignmentVisitor(child,attrname))
        if res is not None:
            return res

def handlePackageScope(package, fqn):
    #print "handlePackageScope",package,fqn
    child = package.getChild(fqn)
    if child:
        return child

    if isinstance(package,Root):
        return getModuleOrPackageUsingFQN(fqn)

    # try searching the fs
    node = getModuleOrPackageUsingFQN(fqn,package.path)
    if node:
        return node
    
    


    # try the package init module
    initmod = package.getChild("__init__")
    if initmod is not None:
        type = getImportedType(initmod, fqn)
        if type:
            return type
    # maybe fqn is absolute
    return getTypeOf(getRoot(), fqn)


wordRE = re.compile("\w+")
def isWordInLine(word, line):
    if line.find(word) != -1:
        words = wordRE.findall(line)
        if word in words:
            return 1
    return 0

def getImportedType(scope, fqn):
    lines = scope.module.getSourceNode().getLines()
    for lineno in scope.getImportLineNumbers():
        logicalline = generateLogicalLines(lines[lineno-1:]).next()
        logicalline = makeLineParseable(logicalline)
        ast = compiler.parse(logicalline)
        match = visitor.walk(ast, ImportVisitor(scope,fqn)).match
        if match:
            return match
        #else loop

class ImportVisitor:
    def __init__(self,scope,fqn):
        self.match = None
        self.targetfqn = fqn
        self.scope = scope

    def visitImport(self, node):
        # if target fqn is an import, then it must be a module or package
        for name, alias in node.names:
            if name == self.targetfqn:
                self.match = resolveImportedModuleOrPackage(self.scope,name)
            elif alias is not None and alias == self.targetfqn:
                self.match = resolveImportedModuleOrPackage(self.scope,name)

    def visitFrom(self, node):
        if node.names[0][0] == '*': # e.g. from foo import *
            if not "."in self.targetfqn:
                module = resolveImportedModuleOrPackage(self.scope,
                                                        node.modname)
                if module:
                    self.match = getTypeOf(module, self.targetfqn)
        else:
            for name, alias in node.names:
                if alias == self.targetfqn or \
                   (alias is None and name == self.targetfqn):
                    scope = resolveImportedModuleOrPackage(self.scope,
                                                            node.modname)
                    if scope is not None:
                        if isinstance(scope,Package):
                            self.match = getModuleOrPackageUsingFQN(name,scope.path)
                        else:  
                            assert isinstance(scope,Module)
                            self.match = getTypeOf(scope, name)




class TypeNotSupportedException:
    def __init__(self,msg):
        self.msg = msg

    def __str__(self):
        return self.msg

# attempts to evaluate the type of the expression
def getTypeOfExpr(scope, ast):
    if isinstance(ast, compiler.ast.Name):
        return getTypeOf(scope, ast.name)

    elif isinstance(ast, compiler.ast.Getattr) or \
             isinstance(ast, compiler.ast.AssAttr):

        # need to do this in order to match foo.bah.baz as
        # a string in import statements
        fqn = attemptToConvertGetattrToFqn(ast)
        if fqn is not None:
            return getTypeOf(scope,fqn)

        expr = getTypeOfExpr(scope, ast.expr)
        if expr is not None:
            attrnametype = getTypeOf(expr, ast.attrname)
            return attrnametype
        return None

    elif isinstance(ast, compiler.ast.CallFunc):
        node = getTypeOfExpr(scope,ast.node)
        if isinstance(node,Class):
            return Instance(node)
        elif isinstance(node,Function):
            return getReturnTypeOfFunction(node)
    else:
        #raise TypeNotSupportedException, \
        #      "Evaluation of "+str(ast)+" not supported. scope="+str(scope)
        print >> log.warning, "Evaluation of "+str(ast)+" not supported. scope="+str(scope)
        return None


def attemptToConvertGetattrToFqn(ast):
    fqn = ast.attrname
    ast = ast.expr
    while isinstance(ast,compiler.ast.Getattr):
        fqn = ast.attrname + "." + fqn
        ast = ast.expr
    if isinstance(ast,compiler.ast.Name):
        return ast.name + "." + fqn
    else:
        return None


getReturnTypeOfFunction_stack = []
def getReturnTypeOfFunction(function):
    if function in getReturnTypeOfFunction_stack:   # loop protection
        return None
    try:
        getReturnTypeOfFunction_stack.append(function)
        return getReturnTypeOfFunction_impl(function)
    finally:
        del getReturnTypeOfFunction_stack[-1]

def getReturnTypeOfFunction_impl(function):
    return scanScopeAST(function,"return",ReturnTypeVisitor(function))
    

# does parse of scope sourcecode to deduce type
def scanScopeSourceForType(scope, name):
    return scanScopeAST(scope,name,AssignmentVisitor(scope,name))


# scans for lines containing keyword, and then runs the visitor over
# the parsed AST for that line
def scanScopeAST(scope,keyword,astvisitor):
    lines = scope.getLinesNotIncludingThoseBelongingToChildScopes()
    src = ''.join(lines)
    match = None
    #print "scanScopeAST:"+str(scope)
    for line in splitLogicalLines(src):
        if isWordInLine(keyword, line):
            #print "scanning for "+keyword+" in line:"+line[:-1]
            doctoredline = makeLineParseable(line)
            ast = compiler.parse(doctoredline)
            match = visitor.walk(ast,astvisitor).getMatch()
            if match:
                return match
    return match
    

class AssignmentVisitor:
    def __init__(self,scope,targetName):
        self.match=None
        self.scope = scope
        self.targetName = targetName

    def getMatch(self):
        return self.match

    def visitAssign(self,node):
        if isinstance(node.expr,compiler.ast.CallFunc):
            for assnode in node.nodes:
                if isinstance(assnode,compiler.ast.AssName) and \
                   assnode.name == self.targetName:
                    self.match = getTypeOfExpr(self.scope,node.expr)
                    if self.match is None:
                        self.match = UnfoundType()
                    
                    

class SelfAttributeAssignmentVisitor:
    def __init__(self,scope,targetName):
        self.match=None
        self.scope = scope
        self.targetName = targetName

    def getMatch(self):
        return self.match

    def visitAssign(self,node):
        if isinstance(node.expr,compiler.ast.CallFunc):
            for assnode in node.nodes:
                if isinstance(assnode,compiler.ast.AssAttr) and \
                   isinstance(assnode.expr,compiler.ast.Name) and \
                   assnode.expr.name == "self" and \
                   assnode.attrname == self.targetName:
                    self.match = getTypeOfExpr(self.scope,node.expr)
                    #print "here!",self.match.getType().fqn


class ReturnTypeVisitor:
    def __init__(self,fn):
        self.match=None
        self.fn = fn

    def getMatch(self):
        return self.match
    
    def visitReturn(self,node):
        try:
            self.match = getTypeOfExpr(self.fn,node.value)
        except TypeNotSupportedException, ex:
            pass


def resolveImportedModuleOrPackage(scope,fqn):
    # try searching from directory containing scope module
    path = os.path.dirname(scope.module.filename)
    node = getModuleOrPackageUsingFQN(fqn,path)
    if node is not None:
        return node
    # try searching from the root
    node = getModuleOrPackageUsingFQN(fqn)
    if node is not None:
        return node
    


--- NEW FILE: findReferences.py ---
from __future__ import generators
from bike.globals import *
from bike.parsing.fastparserast import Module, Class, Function, getRoot, Instance
from bike.query.common import Match, MatchFinder,\
     getScopeForLine, indexToCoordinates, \
     translateSourceCoordsIntoASTNode, scanScopeForMatches,\
     globalScanForMatches, isAMethod, convertNodeToMatchObject
from compiler.ast import AssName,Name,Getattr,AssAttr
import compiler
from findDefinition import findDefinitionFromASTNode
from bike.query.getTypeOf import getTypeOfExpr, UnfoundType
from bike.query.relationships import getRootClassesOfHierarchy
from bike import log
from bike.parsing.load import getSourceNode


class CouldntFindDefinitionException(Exception):
    pass

def findReferencesIncludingDefn(filename,lineno,col):
    return findReferences(filename,lineno,col,1)


def findReferences(filename,lineno,col,includeDefn=0):
    sourcenode = getSourceNode(filename)
    node = translateSourceCoordsIntoASTNode(filename,lineno,col)
    assert node is not None
    scope,defnmatch = getDefinitionAndScope(sourcenode,lineno,node)

    try:
        for match in findReferencesIncludingDefn_impl(sourcenode,node,
                                                      scope,defnmatch):
            if not includeDefn and match == defnmatch: 
                continue        # don't return definition
            else:
                yield match
    except CouldntFindDefinitionException:
        raise CouldntFindDefinitionException("Could not find definition. Please locate manually (maybe using find definition) and find references from that")

def findReferencesIncludingDefn_impl(sourcenode,node,scope,defnmatch):
    if isinstance(node,Name) or isinstance(node,AssName):
        return generateRefsToName(node.name,scope,sourcenode,defnmatch)
    elif isinstance(node,Getattr) or isinstance(node,AssAttr):
        exprtype = getTypeOfExpr(scope,node.expr)
        if exprtype is None or isinstance(exprtype,UnfoundType):
            raise CouldntFindDefinitionException()

        if isinstance(exprtype,Instance):
            exprtype = exprtype.getType()
            return generateRefsToAttribute(exprtype,node.attrname)
        
        else:
            targetname = node.attrname
            return globalScanForMatches(sourcenode.filename,
                                        NameRefFinder(targetname, defnmatch),
                                        targetname, )
        if match is not None:
            return match
    elif isinstance(node,compiler.ast.Function) or \
                             isinstance(node,compiler.ast.Class):
        return handleClassOrFunctionRefs(scope, node, defnmatch)
    else:
        assert 0,"Seed to references must be Name,Getattr,Function or Class"

def handleClassOrFunctionRefs(scope, node, defnmatch):
    if isAMethod(scope,node):        
        for ref in generateRefsToAttribute(scope,node.name):
            yield ref
    else:
        #yield convertNodeToMatchObject(node,100)
        yield defnmatch
        for ref in generateRefsToName(node.name,scope,
                                    scope.module.getSourceNode(),
                                    defnmatch):
            yield ref

def getDefinitionAndScope(sourcenode,lineno,node):
    scope = getScopeForLine(sourcenode,lineno)
    if scope.getStartLine() == lineno and \
           scope.matchesCompilerNode(node):  # scope is the node
        return scope.getParent(), convertNodeToMatchObject(scope,100)
    defnmatch = findDefinitionFromASTNode(scope,node)
    if defnmatch is None:
        raise CouldntFindDefinitionException()
    scope = getScopeForLine(sourcenode,defnmatch.lineno)
    return scope,defnmatch

def generateRefsToName(name,scope,sourcenode,defnmatch):
    assert scope is not None
    if isinstance(scope,Function):
        # search can be limited to scope
        return scanScopeForMatches(sourcenode,scope,
                                   NameRefFinder(name,defnmatch),
                                   name)
    else:        
        return globalScanForMatches(sourcenode.filename,
                                    NameRefFinder(name,defnmatch),
                                    name)


class NameRefFinder(MatchFinder):
    def __init__(self, targetstr,targetMatch):
        self.targetstr = targetstr
        self.targetMatch = targetMatch
        
    def visitName(self, node):
        if node.name == self.targetstr:
            potentualMatch = findDefinitionFromASTNode(self.scope, node)
            if  potentualMatch is not None and \
                   potentualMatch == self.targetMatch:
                self.appendMatch(node.name)
        self.popWordsUpTo(node.name)

    visitAssName = visitName

    def visitFunction(self, node):
        self.popWordsUpTo(node.name)
        for arg, default in self.zipArgs(node.argnames, node.defaults):
            if arg == self.targetstr:
                self.appendMatch(arg)
            self.popWordsUpTo(arg)
            if default is not None:
                self.visit(default)
        self.visit(node.code)


    def visitFrom(self, node):
        for elem in node.modname.split("."):
            self.popWordsUpTo(elem)
            
        for name, alias in node.names:
            if name == self.targetstr:
                if alias is not None:
                    pretendNode = Name(alias)
                else:
                    pretendNode = Name(name)
                if findDefinitionFromASTNode(self.scope, pretendNode) \
                                                    == self.targetMatch:
                    self.appendMatch(name)
            self.popWordsUpTo(name)
            if alias is not None:
                self.popWordsUpTo(alias)


    def visitGetattr(self, node):        
        for c in node.getChildNodes():
            self.visit(c)
        if node.attrname == self.targetstr:
            defn = findDefinitionFromASTNode(self.scope, node)
            if defn is not None and defn == self.targetMatch:
                self.appendMatch(node.attrname)
        self.popWordsUpTo(node.attrname)


    def visitImport(self, node):
        for name, alias in node.names:
            if name.split(".")[-1] == self.targetstr:
                getattr = self.createGetattr(name)
                if findDefinitionFromASTNode(self.scope, getattr) == self.targetMatch:
                    self.appendMatch(self.targetstr)
            for nameelem in name.split("."):
                self.popWordsUpTo(nameelem)
            if alias is not None:
                self.popWordsUpTo(alias)

    
    def createGetattr(self,fqn):
        node = Name(fqn[0])
        for name in fqn.split(".")[1:]:
            node = Getattr(node,name)
        return node
                           
def generateRefsToAttribute(classobj,attrname):
    rootClasses = getRootClassesOfHierarchy(classobj)
    attrRefFinder = AttrbuteRefFinder(rootClasses,attrname)
    for ref in globalScanForMatches(classobj.filename, attrRefFinder, attrname):
        yield ref
    print >>log.progress,"Done"
    

class AttrbuteRefFinder(MatchFinder):
    def __init__(self,rootClasses,targetAttribute):
        self.rootClasses = rootClasses
        self.targetAttributeName = targetAttribute

    
    def visitGetattr(self, node):
        for c in node.getChildNodes():
            self.visit(c)

        if node.attrname == self.targetAttributeName:
            exprtype = getTypeOfExpr(self.scope,node.expr)            

            if isinstance(exprtype,Instance) and \
                 self._isAClassInTheSameHierarchy(exprtype.getType()):
                self.appendMatch(self.targetAttributeName)
            elif isinstance(exprtype,UnfoundType) or \
                 exprtype is None:   # couldn't find type, so not sure
                self.appendMatch(self.targetAttributeName,50)
            else:
                pass # definately not a match
        self.popWordsUpTo(node.attrname)

    visitAssAttr = visitGetattr

    def visitFunction(self,node):  # visit methods
        if node.name == self.targetAttributeName:
            parentScope = self.scope.getParent()
            #print parentScope
            #print self.targetClasses
            if isinstance(parentScope,Class) and \
                   self._isAClassInTheSameHierarchy(parentScope):
                self.appendMatch(node.name)

        for c in node.getChildNodes():
            self.visit(c)
        
    def _isAClassInTheSameHierarchy(self,classobj):
        #return classobj in self.targetClasses
        targetRootClasses = getRootClassesOfHierarchy(classobj)
        for rootclass in self.rootClasses:
            if rootclass in targetRootClasses:
                return True
        return False

--- NEW FILE: relationships.py ---
# queries to do with module/class/function relationships
from __future__ import generators
from bike.globals import *
from getTypeOf import getTypeOf, getTypeOfExpr
from bike.parsing.newstuff import generateModuleFilenamesInPythonPath, generateModuleFilenamesInPackage, getPythonPath
from bike.parsing.pathutils import getPackageBaseDirectory
from bike.query.common import MatchFinder, walkLinesContainingStrings, getScopeForLine
from bike import log
from bike.parsing.fastparserast import Module
import re

def getRootClassesOfHierarchy(klass):
    if klass is None:  # i.e. dont have base class in our ast
        return None
    if klass.getBaseClassNames() == []:  # i.e. is a root class
        return [klass]
    else:
        rootclasses = []
        for base in klass.getBaseClassNames():
            baseclass = getTypeOf(klass,base)
            rootclass = getRootClassesOfHierarchy(baseclass)
            if rootclass is None:  # base class not in our ast
                rootclass = [klass]
            rootclasses+=rootclass
        return rootclasses


--- NEW FILE: __init__.py ---
#
--- NEW FILE: common.py ---
from __future__ import generators
from bike.globals import *
from bike.parsing.fastparserast import getRoot, Function, Class, Module, getModule
from bike.parsing.parserutils import generateLogicalLines, makeLineParseable, UnbalancedBracesException, generateLogicalLinesAndLineNumbers
from bike.parsing.newstuff import getSourceNodesContainingRegex
from bike.parsing import visitor
from bike import log
import compiler
from compiler.ast import Getattr, Name
import re

class Match:
    def __repr__(self):
        return ",".join([self.filename, str(self.lineno), str(self.colno),
                         str(self.confidence)])
    def __eq__(self,other):
        if self is None or other is None:
            return False
        return self.filename == other.filename and \
               self.lineno == other.lineno and \
               self.colno == other.colno

def getScopeForLine(sourceNode, lineno):
    scope = None
    childnodes = sourceNode.getFlattenedListOfFastParserASTNodes()
    if childnodes == []:
        return sourceNode.fastparseroot #module node

    scope = sourceNode.fastparseroot

    for node in childnodes:
        if node.linenum > lineno: break
        scope = node

    if scope.getStartLine() != scope.getEndLine(): # is inline
        while scope.getEndLine() <= lineno:
            scope = scope.getParent()
    return scope



# global from the perspective of 'contextFilename'
def globalScanForMatches(contextFilename, matchFinder, targetname):
    for sourcenode in getSourceNodesContainingRegex(targetname, contextFilename):
        print >> log.progress, "Scanning", sourcenode.filename
        searchscope = sourcenode.fastparseroot
        for match in scanScopeForMatches(sourcenode,searchscope,
                                         matchFinder,targetname):
            yield match


def scanScopeForMatches(sourcenode,scope,matchFinder,targetname):
    lineno = scope.getStartLine()
    for line in generateLogicalLines(scope.getMaskedLines()):
        if line.find(targetname) != -1:
            doctoredline = makeLineParseable(line)
            ast = compiler.parse(doctoredline)
            scope = getScopeForLine(sourcenode, lineno)
            matchFinder.reset(line)
            matchFinder.setScope(scope)
            matches = visitor.walk(ast, matchFinder).getMatches()
            for index, confidence in matches:
                match = Match()
                match.filename = sourcenode.filename
                match.sourcenode = sourcenode
                x, y = indexToCoordinates(line, index)
                match.lineno = lineno+y
                match.colno = x
                match.colend = match.colno+len(targetname)
                match.confidence = confidence
                yield match
        lineno+=line.count("\n")
    

def walkLinesContainingStrings(scope,astWalker,targetnames):
    lineno = scope.getStartLine()
    for line in generateLogicalLines(scope.getMaskedLines()):
        if lineContainsOneOf(line,targetnames):
            doctoredline = makeLineParseable(line)
            ast = compiler.parse(doctoredline)
            astWalker.lineno = lineno
            matches = visitor.walk(ast, astWalker)
        lineno+=line.count("\n")


def lineContainsOneOf(line,targetnames):
    for name in targetnames:
        if line.find(name) != -1:
            return True
    return False


# translates an idx in a logical line into physical line coordinates
# returns x and y coords
def indexToCoordinates(src, index):
    y = src[: index].count("\n")
    startOfLineIdx = src.rfind("\n", 0, index)+1
    x = index-startOfLineIdx
    return x, y



# interface for MatchFinder classes
# implement the visit methods
class MatchFinder:
    def setScope(self, scope):
        self.scope = scope

    def reset(self, line):
        self.matches = []
        self.words = re.split("(\w+)", line) # every other one is a non word
        self.positions = []
        i = 0
        for word in self.words:
            self.positions.append(i)
            #if '\n' in word:  # handle newlines
            #    i = len(word[word.index('\n')+1:])
            #else:
            i+=len(word)
        self.index = 0

    def getMatches(self):
        return self.matches

    # need to visit childnodes in same order as they appear
    def visitPrintnl(self,node):
        if node.dest:
            self.visit(node.dest)
        for n in node.nodes:
            self.visit(n)
    
    def visitName(self, node):
        self.popWordsUpTo(node.name)

    def visitClass(self, node):
        self.popWordsUpTo(node.name)
        for base in node.bases:
            self.visit(base)

    def zipArgs(self, argnames, defaults):
        """Takes a list of argument names and (possibly a shorter) list of
        default values and zips them into a list of pairs (argname, default).
        Defaults are aligned so that the last len(defaults) arguments have
        them, and the first len(argnames) - len(defaults) pairs have None as a
        default.
        """
        fixed_args = len(argnames) - len(defaults)
        defaults = [None] * fixed_args + list(defaults)
        return zip(argnames, defaults)

    def visitFunction(self, node):
        self.popWordsUpTo(node.name)
        for arg, default in self.zipArgs(node.argnames, node.defaults):
            self.popWordsUpTo(arg)
            if default is not None:
                self.visit(default)
        self.visit(node.code)

    def visitGetattr(self,node):
        self.visit(node.expr)
        self.popWordsUpTo(node.attrname)

    def visitAssName(self, node):
        self.popWordsUpTo(node.name)

    def visitAssAttr(self, node):
        self.visit(node.expr)
        self.popWordsUpTo(node.attrname)

    def visitImport(self, node):
        for name, alias in node.names:
            for nameelem in name.split("."):
                self.popWordsUpTo(nameelem)
            if alias is not None:
                self.popWordsUpTo(alias)

    def visitFrom(self, node):
        for elem in node.modname.split("."):
            self.popWordsUpTo(elem)
        for name, alias in node.names:
            self.popWordsUpTo(name)
            if alias is not None:
                self.popWordsUpTo(alias)

    def visitLambda(self, node):
        for arg, default in self.zipArgs(node.argnames, node.defaults):
            self.popWordsUpTo(arg)
            if default is not None:
                self.visit(default)
        self.visit(node.code)

    def visitGlobal(self, node):
        for name in node.names:
            self.popWordsUpTo(name)

    def popWordsUpTo(self, word):
        if word == "*":
            return        # won't be able to find this
        posInWords = self.words.index(word)
        idx = self.positions[posInWords]
        self.words = self.words[posInWords+1:]
        self.positions = self.positions[posInWords+1:]

    def appendMatch(self,name,confidence=100):
        idx = self.getNextIndexOfWord(name)
        self.matches.append((idx, confidence))

    def getNextIndexOfWord(self,name):
        return self.positions[self.words.index(name)]

class CouldNotLocateNodeException(Exception): pass

def translateSourceCoordsIntoASTNode(filename,lineno,col):
    module = getModule(filename)
    maskedlines = module.getMaskedModuleLines()
    lline,backtrackchars = getLogicalLine(module, lineno)
    doctoredline = makeLineParseable(lline)
    ast = compiler.parse(doctoredline)
    idx = backtrackchars+col
    nodefinder = ASTNodeFinder(lline,idx)
    node = compiler.walk(ast, nodefinder).node
    if node is None:
        raise CouldNotLocateNodeException("Could not translate editor coordinates into source node")
    return node

def getLogicalLine(module, lineno):
    # we know that the scope is the start of a logical line, so
    # we search from there
    scope = getScopeForLine(module.getSourceNode(), lineno)
    linegenerator = \
            module.generateLinesWithLineNumbers(scope.getStartLine())
    for lline,llinenum in \
            generateLogicalLinesAndLineNumbers(linegenerator):
        if llinenum > lineno:
            break
        prevline = lline
        prevlinenum = llinenum

    backtrackchars = 0
    for i in range(prevlinenum,lineno):
        backtrackchars += len(module.getSourceNode().getLines()[i-1])
    return prevline, backtrackchars



class ASTNodeFinder(MatchFinder):
    # line is a masked line of text
    # lineno and col are coords
    def __init__(self,line,col):
        self.line = line
        self.col = col
        self.reset(line)
        self.node = None

    def visitName(self,node):
        if self.checkIfNameMatchesColumn(node.name):
            self.node = node
        self.popWordsUpTo(node.name)

    def visitGetattr(self,node):
        self.visit(node.expr)
        if self.checkIfNameMatchesColumn(node.attrname):
            self.node = node
        self.popWordsUpTo(node.attrname)

    def visitFunction(self, node):
        if self.checkIfNameMatchesColumn(node.name):
            self.node = node
        self.popWordsUpTo(node.name)

        for arg, default in self.zipArgs(node.argnames, node.defaults):
            if self.checkIfNameMatchesColumn(arg):
                self.node = Name(arg)
            self.popWordsUpTo(arg)
            if default is not None:
                self.visit(default)
        self.visit(node.code)


    visitAssName = visitName
    visitAssAttr = visitGetattr

    def visitClass(self, node):
        if self.checkIfNameMatchesColumn(node.name):
            self.node = node
        self.popWordsUpTo(node.name)
        for base in node.bases:
            self.visit(base)


    def checkIfNameMatchesColumn(self,name):
        idx = self.getNextIndexOfWord(name)
        #print "name",name,"idx",idx,"self.col",self.col
        if idx <= self.col and idx+len(name) > self.col:
            return 1
        return 0

    def visitFrom(self, node):
        for elem in node.modname.split("."):
            self.popWordsUpTo(elem)
        for name, alias in node.names:
            if self.checkIfNameMatchesColumn(name):
                self.node = self._manufactureASTNodeFromFQN(name)
                return
            self.popWordsUpTo(name)
            if alias is not None:
                self.popWordsUpTo(alias)

    # gets round the fact that imports etc dont contain nested getattr
    # nodes for fqns (e.g. import a.b.bah) by converting the fqn
    # string into a getattr instance
    def _manufactureASTNodeFromFQN(self,fqn):
        if "." in fqn:
            assert 0, "getattr not supported yet"
        else:
            return Name(fqn)

def isAMethod(scope,node):
    return isinstance(node,compiler.ast.Function) and \
           isinstance(scope,Class)

def convertNodeToMatchObject(node,confidence=100):
    m = Match()
    m.sourcenode = node.module.getSourceNode()
    m.filename = node.filename
    if isinstance(node,Module):
        m.lineno = 1
        m.colno = 0
    elif isinstance(node,Class) or isinstance(node,Function):
        m.lineno = node.getStartLine()
        m.colno = node.getColumnOfName()
    m.confidence = confidence
    return m