[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/refactor moveToModule.py,NONE,1.1 utils.py,NO

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/pydev/org.python.pydev/PySrc/ThirdParty/brm/bike/refactor
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9579/PySrc/ThirdParty/brm/bike/refactor

Added Files:
	moveToModule.py utils.py extractVariable.py extractMethod.py 
	inlineVariable.py __init__.py rename.py 
Log Message:
Code completion improvements.
Starting refactoring integration with bicycle repair man.


--- NEW FILE: inlineVariable.py ---
from bike.query.findDefinition import findAllPossibleDefinitionsByCoords
from bike.query.findReferences import findReferences
from bike.parsing.parserutils import maskStringsAndRemoveComments, linecontinueRE
from bike.transformer.undo import getUndoStack
from bike.transformer.save import queueFileToSave
from parser import ParserError
from bike.parsing.load import getSourceNode
import compiler
import re


def inlineLocalVariable(filename, lineno,col):
    sourceobj = getSourceNode(filename)
    return inlineLocalVariable_old(sourceobj, lineno,col)

def inlineLocalVariable_old(sourcenode,lineno,col):
    definition, region, regionlinecount = getLocalVariableInfo(sourcenode, lineno, col)
    addUndo(sourcenode)
    replaceReferences(sourcenode, findReferences(sourcenode.filename, definition.lineno, definition.colno), region)
    delLines(sourcenode, definition.lineno-1, regionlinecount)
    updateSource(sourcenode)

def getLocalVariableInfo(sourcenode, lineno, col):
    definition = findDefinition(sourcenode, lineno, col)
    region, linecount = getRegionToInline(sourcenode, definition)
    return definition, region, linecount

def findDefinition(sourcenode, lineno, col):
    definition = findAllPossibleDefinitionsByCoords(sourcenode.filename,
                                                    lineno,col).next()
    assert definition.confidence == 100    
    return definition

def getRegionToInline(sourcenode, defn):
    line, linecount = getLineAndContinues(sourcenode, defn.lineno)
    start, end = findRegionToInline(maskStringsAndRemoveComments(line))
    return line[start:end], linecount

def findRegionToInline(maskedline):
    match = re.compile("[^=]+=\s*(.+)$\n", re.DOTALL).match(maskedline)
    assert match
    return match.start(1), match.end(1)

# Possible refactoring: move to class of sourcenode
def getLineAndContinues(sourcenode, lineno):
    line = sourcenode.getLine(lineno)
    
    linecount = 1
    while linecontinueRE.search(line):
        line += sourcenode.getLine(lineno + linecount)
        linecount += 1

    return line, linecount

def addUndo(sourcenode):
    getUndoStack().addSource(sourcenode.filename,sourcenode.getSource())

def replaceReferences(sourcenode, references, replacement):
    for reference in safeReplaceOrder( references ):
        replaceReference(sourcenode, reference, replacement)

def safeReplaceOrder( references ):
    """ 
    When inlining a variable, if multiple instances occur on the line, then the
    last reference must be replaced first. Otherwise the remaining intra-line
    references will be incorrect.
    """
    def safeReplaceOrderCmp(self, other):
        return -cmp(self.colno, other.colno)

    result = list(references)
    result.sort(safeReplaceOrderCmp)
    return result


def replaceReference(sourcenode, ref, replacement):
    """ sourcenode.getLines()[ref.lineno-1][ref.colno:ref.colend] = replacement
    But strings don't support slice assignment as they are immutable. :(
    """
    sourcenode.getLines()[ref.lineno-1] = \
        replaceSubStr(sourcenode.getLines()[ref.lineno-1],
            ref.colno, ref.colend, replacement)

def replaceSubStr(str, start, end, replacement):
    return str[:start] + replacement + str[end:]

# Possible refactoring: move to class of sourcenode
def delLines(sourcenode, lineno, linecount=1):
    del sourcenode.getLines()[lineno:lineno+linecount]
    
def updateSource(sourcenode):
    queueFileToSave(sourcenode.filename,"".join(sourcenode.getLines()))
    
                    
--- NEW FILE: extractVariable.py ---
from bike.parsing.parserutils import maskStringsAndRemoveComments
from bike.transformer.undo import getUndoStack
from parser import ParserError
import compiler
from bike.refactor.extractMethod import coords
from bike.refactor.utils import getTabWidthOfLine, getLineSeperator,\
     reverseCoordsIfWrongWayRound
from bike.transformer.save import queueFileToSave
from bike.parsing.load import getSourceNode


def extractLocalVariable(filename, startcoords, endcoords, varname):
    sourceobj = getSourceNode(filename)
    if startcoords.line != endcoords.line:
        raise "Can't do multi-line extracts yet"
    startcoords, endcoords = \
                 reverseCoordsIfWrongWayRound(startcoords,endcoords)
    line = sourceobj.getLine(startcoords.line)
    tabwidth = getTabWidthOfLine(line)
    linesep = getLineSeperator(line)
    region = line[startcoords.column:endcoords.column]
    
    getUndoStack().addSource(sourceobj.filename,sourceobj.getSource())
    sourceobj.getLines()[startcoords.line-1] = \
          line[:startcoords.column] + varname + line[endcoords.column:]

    defnline = tabwidth*" " + varname + " = " + region + linesep
    
    sourceobj.getLines().insert(startcoords.line-1,defnline)

    queueFileToSave(sourceobj.filename,"".join(sourceobj.getLines()))


--- NEW FILE: extractMethod.py ---
import re
import compiler
from bike.parsing import visitor
from bike.query.common import getScopeForLine
from bike.parsing.parserutils import generateLogicalLines, \
                makeLineParseable, maskStringsAndRemoveComments
from parser import ParserError
from bike.parsing.fastparserast import Class
from bike.transformer.undo import getUndoStack
from bike.refactor.utils import getTabWidthOfLine, getLineSeperator, \
                reverseCoordsIfWrongWayRound
from bike.transformer.save import queueFileToSave
from bike.parsing.load import getSourceNode
TABSIZE = 4

class coords:
    def __init__(self, line, column):
        self.column = column
        self.line = line
    def __str__(self):
        return "("+str(self.column)+","+str(self.line)+")"

commentRE = re.compile(r"#.*?$")

class ParserException(Exception): pass

def extractMethod(filename, startcoords, endcoords, newname):
    ExtractMethod(getSourceNode(filename),
                  startcoords, endcoords, newname).execute()

class ExtractMethod(object):
    def __init__(self,sourcenode, startcoords, endcoords, newname):
        self.sourcenode = sourcenode

        startcoords, endcoords = \
               reverseCoordsIfWrongWayRound(startcoords,endcoords)

        self.startline = startcoords.line
        self.endline = endcoords.line
        self.startcol = startcoords.column
        self.endcol= endcoords.column

        self.newfn = NewFunction(newname)

        self.getLineSeperator()
        self.adjustStartColumnIfLessThanTabwidth()
        self.adjustEndColumnIfStartsANewLine()
        self.fn = self.getFunctionObject()
        self.getRegionToBuffer()
        #print "-"*80
        #print self.extractedLines
        #print "-"*80
        self.deduceIfIsMethodOrFunction()

    def execute(self):
        self.deduceArguments()
        getUndoStack().addSource(self.sourcenode.filename,
                                 self.sourcenode.getSource())
        srclines = self.sourcenode.getLines()
        newFnInsertPosition = self.fn.getEndLine()-1
        self.insertNewFunctionIntoSrcLines(srclines, self.newfn,
                                           newFnInsertPosition)
        self.writeCallToNewFunction(srclines)

        src = "".join(srclines)
        queueFileToSave(self.sourcenode.filename,src)        

    def getLineSeperator(self):
        line = self.sourcenode.getLines()[self.startline-1]
        linesep = getLineSeperator(line)
        self.linesep = linesep        

    def adjustStartColumnIfLessThanTabwidth(self):
        tabwidth = getTabWidthOfLine(self.sourcenode.getLines()[self.startline-1])
        if self.startcol < tabwidth: self.startcol = tabwidth

    def adjustEndColumnIfStartsANewLine(self):
        if self.endcol == 0:
            self.endline -=1
            nlSize = len(self.linesep)
            self.endcol = len(self.sourcenode.getLines()[self.endline-1])-nlSize


    def getFunctionObject(self):
        return getScopeForLine(self.sourcenode,self.startline)


    def getTabwidthOfParentFunction(self):
        line = self.sourcenode.getLines()[self.fn.getStartLine()-1]
        match = re.match("\s+",line)
        if match is None:
            return 0
        else:
            return match.end(0)

    # should be in the transformer module
    def insertNewFunctionIntoSrcLines(self,srclines,newfn,insertpos):
        tabwidth = self.getTabwidthOfParentFunction()

        while re.match("\s*"+self.linesep,srclines[insertpos-1]):
            insertpos -= 1

        srclines.insert(insertpos, self.linesep)
        insertpos +=1

        fndefn = "def "+newfn.name+"("

        if self.isAMethod:
            fndefn += "self"
            if newfn.args != []:
                fndefn += ", "+", ".join(newfn.args)
        else:
            fndefn += ", ".join(newfn.args)

        fndefn += "):"+self.linesep


        srclines.insert(insertpos,tabwidth*" "+fndefn)
        insertpos +=1

        tabwidth += TABSIZE


        if self.extractedCodeIsAnExpression(srclines):
            assert len(self.extractedLines) == 1

            fnbody = [tabwidth*" "+ "return "+self.extractedLines[0]]


        else:
            fnbody = [tabwidth*" "+line for line in self.extractedLines]
            if newfn.retvals != []:
                fnbody.append(tabwidth*" "+"return "+
                             ", ".join(newfn.retvals) + self.linesep)

        for line in fnbody:
            srclines.insert(insertpos,line)
            insertpos +=1


    def writeCallToNewFunction(self, srclines):
        startline = self.startline
        endline = self.endline
        startcol = self.startcol
        endcol= self.endcol

        fncall = self.constructFunctionCallString(self.newfn.name, self.newfn.args,
                                                  self.newfn.retvals)

        self.replaceCodeWithFunctionCall(srclines, fncall,
                                         startline, endline, startcol, endcol)


    def replaceCodeWithFunctionCall(self, srclines, fncall,
                                    startline, endline, startcol, endcol):
        if startline == endline:  # i.e. extracted code part of existing line
            line = srclines[startline-1]
            srclines[startline-1] = self.replaceSectionOfLineWithFunctionCall(line,
                                                         startcol, endcol, fncall)
        else:
            self.replaceLinesWithFunctionCall(srclines, startline, endline, fncall)


    def replaceLinesWithFunctionCall(self, srclines, startline, endline, fncall):
        tabwidth = getTabWidthOfLine(srclines[startline-1])
        line = tabwidth*" " + fncall + self.linesep
        srclines[startline-1:endline] = [line]


    def replaceSectionOfLineWithFunctionCall(self, line, startcol, endcol, fncall):
        line = line[:startcol] + fncall + line[endcol:]
        if not line.endswith(self.linesep):
            line+=self.linesep
        return line


    def constructFunctionCallString(self, fnname, fnargs, retvals):
        fncall = fnname + "("+", ".join(fnargs)+")"
        if self.isAMethod:
            fncall = "self." + fncall

        if retvals != []:
            fncall = ", ".join(retvals) + " = "+fncall
        return fncall


    def deduceArguments(self):
        lines = self.fn.getLinesNotIncludingThoseBelongingToChildScopes()

        # strip off comments
        lines = [commentRE.sub(self.linesep,line) for line in lines]
        extractedLines = maskStringsAndRemoveComments("".join(self.extractedLines)).splitlines(1)

        linesbefore = lines[:(self.startline - self.fn.getStartLine())]
        linesafter = lines[(self.endline - self.fn.getStartLine()) + 1:]

        # split into logical lines
        linesbefore = [line for line in generateLogicalLines(linesbefore)]        
        extractedLines = [line for line in generateLogicalLines(extractedLines)]
        linesafter = [line for line in generateLogicalLines(linesafter)]

        if self.startline == self.endline:
            # need to include the line code is extracted from
            line = generateLogicalLines(lines[self.startline - self.fn.getStartLine():]).next()
            linesbefore.append(line[:self.startcol] + "dummyFn()" + line[self.endcol:])
        assigns = getAssignments(linesbefore)
        fnargs = getFunctionArgs(linesbefore)
        candidateArgs = assigns + fnargs            
        refs = getVariableReferencesInLines(extractedLines)
        self.newfn.args = [ref for ref in refs if ref in candidateArgs]

        assignsInExtractedBlock = getAssignments(extractedLines)
        usesAfterNewFunctionCall = getVariableReferencesInLines(linesafter)
        usesInPreceedingLoop = getVariableReferencesInLines(
            self.getPreceedingLinesInLoop(linesbefore,line))
        self.newfn.retvals = [ref for ref in usesInPreceedingLoop+usesAfterNewFunctionCall
                                   if ref in assignsInExtractedBlock]

    def getPreceedingLinesInLoop(self,linesbefore,firstLineToExtract):
        if linesbefore == []: return []
        tabwidth = getTabWidthOfLine(firstLineToExtract)
        rootTabwidth = getTabWidthOfLine(linesbefore[0])
        llines = [line for line in generateLogicalLines(linesbefore)]
        startpos = len(llines)-1
        loopTabwidth = tabwidth
        for idx in range(startpos,0,-1):
            line = llines[idx]
            if re.match("(\s+)for",line) is not None or \
               re.match("(\s+)while",line) is not None:
                candidateLoopTabwidth = getTabWidthOfLine(line)
                if candidateLoopTabwidth < loopTabwidth:
                    startpos = idx
        return llines[startpos:]

    
    def getRegionToBuffer(self):
        startline = self.startline
        endline = self.endline
        startcol = self.startcol
        endcol= self.endcol


        self.extractedLines = self.sourcenode.getLines()[startline-1:endline]

        match = re.match("\s*",self.extractedLines[0])
        tabwidth = match.end(0)

        self.extractedLines = [line[startcol:] for line in self.extractedLines]

        # above cropping can take a blank line's newline off.
        # this puts it back
        for idx in range(len(self.extractedLines)):
            if self.extractedLines[idx] == '':
                self.extractedLines[idx] = self.linesep

        if startline == endline:
            # need to crop the end
            # (n.b. if region is multiple lines, then whole lines are taken)
            self.extractedLines[-1] = self.extractedLines[-1][:endcol-startcol]

        if self.extractedLines[-1][-1] != '\n':
            self.extractedLines[-1] += self.linesep

    def extractedCodeIsAnExpression(self,lines):
        if len(self.extractedLines) == 1:
            charsBeforeSelection = lines[self.startline-1][:self.startcol]
            if re.match("^\s*$",charsBeforeSelection) is not None:
                return 0
            if re.search(":\s*$",charsBeforeSelection) is not None:
                return 0
            return 1
        return 0

    def deduceIfIsMethodOrFunction(self):
        if isinstance(self.fn.getParent(),Class):
            self.isAMethod = 1
        else:
            self.isAMethod = 0


# holds information about the new function
class NewFunction:
    def __init__(self,name):
        self.name = name


# lines = list of lines.
# Have to have strings masked and comments removed
def getAssignments(lines):
    class AssignVisitor:
        def __init__(self):
            self.assigns = []

        def visitAssTuple(self, node):
            for a in node.nodes:
                if a.name not in self.assigns:
                    self.assigns.append(a.name)

        def visitAssName(self, node):
            if node.name not in self.assigns:
                self.assigns.append(node.name)

        def visitAugAssign(self, node):
            if isinstance(node.node, compiler.ast.Name):
                if node.node.name not in self.assigns:
                    self.assigns.append(node.node.name)

    assignfinder = AssignVisitor()
    for line in lines:
        doctoredline = makeLineParseable(line)
        try:
            ast = compiler.parse(doctoredline)
        except ParserError:
            raise ParserException("couldnt parse:"+doctoredline)
        visitor.walk(ast, assignfinder)
    return assignfinder.assigns


# lines = list of lines.
# Have to have strings masked and comments removed
def getFunctionArgs(lines):
    if lines == []: return []

    class FunctionVisitor:
        def __init__(self):
            self.result = []
        def visitFunction(self, node):
            for n in node.argnames:
                if n != "self":
                    self.result.append(n)
    fndef = generateLogicalLines(lines).next()
    doctoredline = makeLineParseable(fndef)
    try:
        ast = compiler.parse(doctoredline)
    except ParserError:
        raise ParserException("couldnt parse:"+doctoredline)
    return visitor.walk(ast, FunctionVisitor()).result


# lines = list of lines. Have to have strings masked and comments removed
def getVariableReferencesInLines(lines):
    class NameVisitor:
        def __init__(self):
            self.result = []
        def visitName(self, node):
            if node.name not in self.result:
                self.result.append(node.name)
    reffinder = NameVisitor()
    for line in lines:
        doctoredline = makeLineParseable(line)
        try:
            ast = compiler.parse(doctoredline)
        except ParserError:
            raise ParserException("couldnt parse:"+doctoredline)
        visitor.walk(ast, reffinder)
    return reffinder.result

--- NEW FILE: rename.py ---
from bike.transformer.WordRewriter import WordRewriter
from bike.query.findReferences import findReferencesIncludingDefn
from bike.transformer.save import save

def rename(filename,lineno,col,newname,promptcallback=None):
    strrewrite = WordRewriter()
    for match in findReferencesIncludingDefn(filename,lineno,col):
        #print "rename match ",match
        if match.confidence == 100 or promptUser(promptcallback,match):
            strrewrite.rewriteString(match.sourcenode,
                                     match.lineno,match.colno,newname)
    strrewrite.commit()

def promptUser(promptCallback,match):
    if promptCallback is not None and \
       promptCallback(match.filename, match.lineno, match.colno, match.colend):
        return 1
    return 0
    

--- NEW FILE: utils.py ---
import re

def getLineSeperator(line):
    if line.endswith("\r\n"):
        linesep = "\r\n"          # windoze
    else:
        linesep = line[-1]   # mac or unix
    return linesep


def getTabWidthOfLine(line):
    match = re.match("\s+",line)
    if match is None:
        return 0
    else:
        return match.end(0)

def reverseCoordsIfWrongWayRound(startcoords,endcoords):
    if(startcoords.line > endcoords.line) or \
         (startcoords.line == endcoords.line and \
          startcoords.column > endcoords.column):
        return endcoords,startcoords
    else:
        return startcoords,endcoords

--- NEW FILE: moveToModule.py ---
import bike.globals
from bike.parsing.load import getSourceNode
from bike.parsing.fastparserast import Module
from bike.query.common import getScopeForLine, convertNodeToMatchObject
from bike.transformer.save import queueFileToSave, save
from bike.transformer.undo import getUndoStack
from bike.refactor.extractMethod import getVariableReferencesInLines
from bike.refactor.utils import getLineSeperator
from bike.query.findDefinition import findDefinitionFromASTNode
from bike.query.findReferences import findReferences
from bike.parsing.pathutils import filenameToModulePath
from compiler.ast import Name
import re

def moveClassToNewModule(origfile,line,newfile):
    srcnode = getSourceNode(origfile)
    targetsrcnode = getSourceNode(newfile)
    classnode = getScopeForLine(srcnode,line)
    classlines = srcnode.getLines()[classnode.getStartLine()-1:
                                    classnode.getEndLine()-1]
    getUndoStack().addSource(srcnode.filename,
                             srcnode.getSource())
    getUndoStack().addSource(targetsrcnode.filename,
                             targetsrcnode.getSource())

    srcnode.getLines()[classnode.getStartLine()-1:
                       classnode.getEndLine()-1] = []
    
    targetsrcnode.getLines().extend(classlines)

    queueFileToSave(srcnode.filename,srcnode.getSource())
    queueFileToSave(targetsrcnode.filename,targetsrcnode.getSource())


exactFromRE = "(from\s+\S+\s+import\s+%s)(.*)"    
fromRE = "from\s+\S+\s+import\s+(.*)"

def moveFunctionToNewModule(origfile,line,newfile):
    
    srcnode = getSourceNode(origfile)
    targetsrcnode = getSourceNode(newfile)
    scope = getScopeForLine(srcnode,line)

    linesep = getLineSeperator(srcnode.getLines()[0])

    matches = findReferences(origfile, line, scope.getColumnOfName())

    origFileImport = []
    fromline = 'from %s import %s'%(filenameToModulePath(newfile),scope.name)
    
    for match in matches:
        if match.filename == origfile:
            origFileImport = fromline + linesep
        else:
            s = getSourceNode(match.filename)
            m = s.fastparseroot
            if match.lineno in m.getImportLineNumbers():                
                getUndoStack().addSource(s.filename,
                                         s.getSource())

                maskedline = m.getLogicalLine(match.lineno)
                origline = s.getLines()[match.lineno-1]
                reMatch =  re.match(exactFromRE%(scope.name),maskedline)
                if reMatch and not (',' in reMatch.group(2) or \
                                    '\\' in reMatch.group(2)):
                    restOfOrigLine = origline[len(reMatch.group(1)):]
                    s.getLines()[match.lineno-1] = fromline + restOfOrigLine

                elif re.match(fromRE,maskedline):
                    #remove the element from the import stmt
                    line = re.sub('%s\s*?,'%(scope.name),'',origline)
                    s.getLines()[match.lineno-1] = line
                    #and add a new line
                    nextline = match.lineno + maskedline.count('\\') + 1
                    s.getLines()[nextline-1:nextline-1] = [fromline+linesep]
                    
                queueFileToSave(s.filename,s.getSource())
                    
    
    refs = getVariableReferencesInLines(scope.getMaskedLines())

    scopeLines = srcnode.getLines()[scope.getStartLine()-1:
                                    scope.getEndLine()-1]
    importModules = deduceImportsForNewFile(refs, scope)
    importlines = composeNewFileImportLines(importModules, linesep)


    getUndoStack().addSource(srcnode.filename,
                             srcnode.getSource())
    getUndoStack().addSource(targetsrcnode.filename,
                             targetsrcnode.getSource())

    srcnode.getLines()[scope.getStartLine()-1:
                       scope.getEndLine()-1] = origFileImport

    targetsrcnode.getLines().extend(importlines+scopeLines)

    queueFileToSave(srcnode.filename,srcnode.getSource())
    queueFileToSave(targetsrcnode.filename,targetsrcnode.getSource())


def composeNewFileImportLines(importModules, linesep):
    importlines = []
    for mpath in importModules:
        importlines += "from %s import %s"%(mpath,
                                  ', '.join(importModules[mpath]))
        importlines += linesep
    return importlines

def deduceImportsForNewFile(refs, scope):
    importModules = {}
    for ref in refs:
        match = findDefinitionFromASTNode(scope,Name(ref))

        if match.filename == scope.module.filename:
            tgtscope = getScopeForLine(getSourceNode(match.filename),
                                       match.lineno)
            while tgtscope != scope and not isinstance(tgtscope,Module):
                tgtscope = tgtscope.getParent()
            
            if not isinstance(tgtscope,Module):
                continue   # was defined in this function
        
        mpath = filenameToModulePath(match.filename)            
        if mpath in importModules:            
            importModules[mpath].append(ref)
        else:
            importModules[mpath] = [ref]
    return importModules
    
    
--- NEW FILE: __init__.py ---

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/refactor moveToModule.py,NONE,1.1 utils.py,NO

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/refactor moveToModule.py,NONE,1.1 utils.py,NONE,1.1 extractVariable.py,NONE,1.1 extractMethod.py,NONE,1.1 inlineVariable.py,NONE,1.1 __init__.py,NONE,1.1 rename.py,NONE,1.1

[Pydev-cvs] org.python.pydev/PySrc/ThirdParty/brm/bike/refactor moveToModule.py,NONE,1.1 utils.py,NONE,1.1 extractVariable.py,NONE,1.1 extractMethod.py,NONE,1.1 inlineVariable.py,NONE,1.1 init.py,NONE,1.1 rename.py,NONE,1.1