[Cpptool-cvs] CppParser/src/pyrfta symboltabletest.py,NONE,1.1 testtools.py,NONE,1.1 pycppparser.py,1.1,1.2 pycppparsertest.py,1.1,1.2

[Baptiste L. <bl...@us...>]

Update of /cvsroot/cpptool/CppParser/src/pyrfta
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16971/src/pyrfta

Modified Files:
	pycppparser.py pycppparsertest.py 
Added Files:
	symboltabletest.py testtools.py 
Log Message:
* ported the basis of symbol table test

--- NEW FILE: symboltabletest.py ---
import testtools
import unittest
import os.path
from pycppparser import *

class DeclarationsNodeProcessor(ValueNodeProcessor):
    def __init__( self ):
        ValueNodeProcessor.__init__( self )

        globalDeclarations = NodeSpecification( "global_declarations", "", NS_HAS_CHILDREN )
        globalDeclarations.addChild( "aggregate" )
        globalDeclarations.addChild( "type_alias" )
        globalDeclarations.addChild( "namespace" )
        globalDeclarations.addChild( "anonymous_namespace" )
        globalDeclarations.addChild( "variable" )
        globalDeclarations.addChild( "enumeration" )
        self.recordSpecification( globalDeclarations )
        self.setRootSpecification( globalDeclarations.specificationName )

        declarations = NodeSpecification( "declarations", "declarations", NS_HAS_COLON_STRING_LIST_VALUE )
        self.recordSpecification( declarations )

        references = NodeSpecification( "references", "references", NS_HAS_COLON_STRING_LIST_VALUE )
        self.recordSpecification( references )

        aggregate = NodeSpecification( "aggregate", "aggregate", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        aggregate.addChild( "aggregate_kind" )
        aggregate.addMandatoryChild( "declarations" )
        aggregate.addMandatoryChild( "references" )
        aggregate.addChild( "aggregate_members" )
        self.recordSpecification( aggregate )

        aggregateKind = NodeSpecification( "aggregate_kind", "kind", NS_HAS_COLON_ENUMERATION_VALUE )
        aggregateKind.addEnumerationValue( "class" )
        aggregateKind.addEnumerationValue( "struct" )
        aggregateKind.addEnumerationValue( "union" )
        self.recordSpecification( aggregateKind )

        aggregateMembers = NodeSpecification( "aggregate_members", "members", NS_HAS_CHILDREN )
        aggregateMembers.addChild( "aggregate" )
        aggregateMembers.addChild( "constructor" )
        aggregateMembers.addChild( "destructor" )
        aggregateMembers.addChild( "function" )
        aggregateMembers.addChild( "type_alias" )
        self.recordSpecification( aggregateMembers )

        constructor = NodeSpecification( "constructor", "constructor", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        constructor.addMandatoryChild( "prototypes" )
        constructor.addChild( "function_definition" )
        constructor.addChild( "function_parameters" )
        self.recordSpecification( constructor )

        destructor = NodeSpecification( "destructor", "destructor", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        destructor.addMandatoryChild( "prototypes" )
        destructor.addChild( "function_definition" )
        destructor.addChild( "function_parameters" )
        self.recordSpecification( destructor )

        function = NodeSpecification( "function", "function", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        function.addMandatoryChild( "prototypes" )
        function.addMandatoryChild( "return_type" )
        function.addChild( "function_definition" )
        function.addChild( "function_parameters" )
        self.recordSpecification( function )

        prototypes = NodeSpecification( "prototypes", "prototypes", NS_HAS_COLON_STRING_LIST_VALUE )
        self.recordSpecification( prototypes )

        functionDefinition = NodeSpecification( "function_definition", "definition", NS_HAS_COLON_STRING_LIST_VALUE )
        self.recordSpecification( functionDefinition )

        returnType = NodeSpecification( "return_type", "return_type", NS_HAS_COLON_NODE_VALUE )
        returnType.addChild( "type_built_in" )
        returnType.addChild( "type_const" )
        returnType.addChild( "type_volatile" )
        returnType.addChild( "type_ref" )
        returnType.addChild( "type_ptr" )
        self.recordSpecification( returnType )

        functionParameters = NodeSpecification( "function_parameters", "parameters", NS_HAS_CHILDREN )
        functionParameters.addChild( "function_parameter" )
        self.recordSpecification( functionParameters )

        functionParameter = NodeSpecification( "function_parameter", "parameter", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        functionParameter.addMandatoryChild( "type" )
        self.recordSpecification( functionParameter )

        type = NodeSpecification( "type", "type", NS_HAS_COLON_NODE_VALUE )
        type.addChild( "type_built_in" )
        type.addChild( "type_const" )
        type.addChild( "type_volatile" )
        type.addChild( "type_ref" )
        type.addChild( "type_ptr" )
        type.addChild( "type_type" )
        self.recordSpecification( type )

        typeBuiltIn = NodeSpecification( "type_built_in", "built_in", NS_HAS_BRACE_VALUES )
        self.recordSpecification( typeBuiltIn )

        typeType = NodeSpecification( "type_type", "type", NS_HAS_BRACE_VALUE )
        self.recordSpecification( typeType )

        typeConst = NodeSpecification( "type_const", "const", NS_HAS_BRACE_NODE_VALUE )
        typeConst.addChild( "type_built_in" )
        typeConst.addChild( "type_volatile" )
        typeConst.addChild( "type_ref" )
        typeConst.addChild( "type_ptr" )
        typeConst.addChild( "type_type" )
        self.recordSpecification( typeConst )

        typeVolatile = NodeSpecification( "type_volatile", "volatile", NS_HAS_BRACE_NODE_VALUE )
        typeVolatile.addChild( "type_built_in" )
        typeVolatile.addChild( "type_const" )
        typeVolatile.addChild( "type_ref" )
        typeVolatile.addChild( "type_ptr" )
        typeVolatile.addChild( "type_type" )
        self.recordSpecification( typeVolatile )

        typeRef = NodeSpecification( "type_ref", "ref", NS_HAS_BRACE_NODE_VALUE )
        typeRef.addChild( "type_built_in" )
        typeRef.addChild( "type_const" )
        typeRef.addChild( "type_volatile" )
        typeRef.addChild( "type_ptr" )
        typeRef.addChild( "type_type" )
        self.recordSpecification( typeRef )

        typePtr = NodeSpecification( "type_ptr", "ptr", NS_HAS_BRACE_NODE_VALUE )
        typePtr.addChild( "type_built_in" )
        typePtr.addChild( "type_const" )
        typePtr.addChild( "type_volatile" )
        typePtr.addChild( "type_ref" )
        typePtr.addChild( "type_ptr" )
        typePtr.addChild( "type_type" )
        self.recordSpecification( typePtr )

        typeAlias = NodeSpecification( "type_alias", "type_alias", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        typeAlias.addMandatoryChild( "declarations" )
        typeAlias.addMandatoryChild( "references" )
        typeAlias.addMandatoryChild( "type" )
        self.recordSpecification( typeAlias )

        namespaceSpec = NodeSpecification( "namespace", "namespace", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        namespaceSpec.addMandatoryChild( "declarations" )
        namespaceSpec.addMandatoryChild( "references" )
        namespaceSpec.addMandatoryChild( "namespace_members" )
        self.recordSpecification( namespaceSpec )

        anonymousNamespace = NodeSpecification( "anonymous_namespace", "anonymous_namespace", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        anonymousNamespace.addMandatoryChild( "declarations" )
        anonymousNamespace.addMandatoryChild( "namespace_members" )
        self.recordSpecification( anonymousNamespace )

        namespaceMembers = NodeSpecification( "namespace_members", "members", NS_HAS_CHILDREN )
        # .. add declaration sequence members
        namespaceMembers.addChild( "aggregate" )
        namespaceMembers.addChild( "type_alias" )
        namespaceMembers.addChild( "namespace" )
        namespaceMembers.addChild( "variable" )
        namespaceMembers.addChild( "enumeration" )
        self.recordSpecification( namespaceMembers )

        enumeration = NodeSpecification( "enumeration", "enumeration", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        enumeration.addMandatoryChild( "declarations" )
        enumeration.addMandatoryChild( "references" )
        enumeration.addMandatoryChild( "enumeration_values" )
        self.recordSpecification( enumeration )

        enumerationValues = NodeSpecification( "enumeration_values", "values", NS_HAS_CHILDREN )
        enumerationValues.addChild( "enumeration_value" )
        self.recordSpecification( enumerationValues )

        enumerationValue = NodeSpecification( "enumeration_value", "value", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        enumerationValue.addMandatoryChild( "references" )
        self.recordSpecification( enumerationValue )

        variable = NodeSpecification( "variable", "variable", NS_HAS_BRACE_VALUE | NS_HAS_CHILDREN )
        variable.addMandatoryChild( "declarations" )
        variable.addMandatoryChild( "references" )
        variable.addMandatoryChild( "type" )
        self.recordSpecification( variable )

        self.checkSpecifications()
                
        

class SymbolTableTestProcessor(testtools.LineProcessor):
    def __init__( self, location_tracker, filesystem ):
        testtools.LineProcessor.__init__( self, location_tracker )
        self.buffer_name = None
        self.filesystem = filesystem
        self.declarations_start_line = -1
        self.description = ''

    def _commentAllowed( self ):
        pass

    def _doProcessLine( self, line ):
        if not self.buffer_name:
            return
        value = getattr( self, self.buffer_name ) + line
        setattr( self, self.buffer_name, value )

    def _doStateChange( self, name, parameter ):
        if name == 'declarations':
            self.buffer_name = 'declarations'
            self.declarations = ''
            self.declarations_start_line = self.currentLine() + 1
        elif name == 'description':
            self.buffer_name = 'description'
        else:
            return False
        return True

    def _doProcessReadFile( self, path, content ):
        file_id = self.filesystem.setFileContent( path, content )
        self._memorizePendingLocations( file_id )

    def _doTerminateCurrentAction( self ):
        if self.buffer_name == 'declarations':
            self._processDeclarations()
        self.buffer_name = None

    def _processDeclarations( self ):
        processor = DeclarationsNodeProcessor()
        self.declarations_tree = processor.parseNodeTree( self.declarations,
                                                          self.declarations_start_line )

class DeclarationChecker:
    def __init__( self, filesystem, tracker ):
        pass

    def checkDeclarations( self, root ):
        pass

class SymbolTableTestCase(unittest.TestCase):
    def __init__( self, base_path, filename ):
        unittest.TestCase.__init__( self )
        self.path = os.path.join( base_path, filename )

    def runTest( self ):
        tracker = testtools.LocationTracker()
        filesystem = testtools.FileSystem()
        processor = SymbolTableTestProcessor( tracker, filesystem )
        processor.processFile( self.path )
        root = processor.declarations_tree

        checker = DeclarationChecker( filesystem, tracker )
        checker.checkDeclarations( root )

class SymbolTableTestSuite(unittest.TestSuite):
    def __init__( self ):
        unittest.TestSuite.__init__( self )
        generator = testtools.TestListTestCaseGenerator( self,
                                                         SymbolTableTestCase,
                                                         'testdata/symbol_table',
                                                         '__tests__.txt' )

if __name__ == '__main__':
    unittest.main( defaultTest = 'SymbolTableTestSuite' )
 
--- NEW FILE: testtools.py ---
import os.path
import unittest

class LineProcessorError(RuntimeError):
    pass

class LocationTracker:
    def __init__( self ):
        self.locations_by_files = {}

    def memorizeLocation( self, file_id, locations ):
        self.locations_by_files[ file_id ] = locations.copy()

class LineProcessor:
    def __init__( self, location_tracker ):
        self.location_tracker = location_tracker
        self.line = 0
        self.is_reading_file = False

    def processFile( self, path ):
        try:
            f = file(path, 'rt')
            for line in f:
                self.processLine( line )
            f.close()
            self.terminateCurrentAction()
        except IOError, e:
            self._raiseError( 'Failed to read: %s\n%s' % (path,str(e)) )

    def processLine( self, line ):
        self.line += 1
        if self._isCommentLine( line ) or self._handleStateChangeLine( line ):
            return
        if self.is_reading_file:
            self._processReadFile( line )
        else:
            self._doProcessLine( line )

    def terminateCurrentAction( self ):
        if self.is_reading_file:
            self._doProcessReadFile( self.file_path, self.file_content )
            self.is_reading_file = False
        self._doTerminateCurrentAction()

    def currentLine( self ):
        return self.line

    def _memorizePendingLocations( self, file ):
        self.location_tracker.memorizeLocation( file, self.pending_locations )

    def _processActionLine( self, line ):
        pass

    def _isCommentLine( self, line ):
        return self._commentAllowed() and line.startswith( '##' )

    def _handleStateChangeLine( self, line ):
        marker = '===='
        if not line.startswith( marker ):
            return False

        endNameIndex = line.find( ':', len(marker) )
        if endNameIndex == -1:
            self._raiseError( "Missing ':' on state change line." )
        stateName = line[len(marker):endNameIndex]
        parameter = line[endNameIndex:].strip()
        self.terminateCurrentAction()

        if stateName == 'file':
            self._beginReadingFile( parameter )
            return True
        elif stateName == 'end':
            return True

        if not self._doStateChange( stateName, parameter ):
            self._raiseError( "Unknown action: '%s'." % stateName )
        return True

    def _raiseError( self, message ):
        raise LineProcessorError( 'Line: %d\n%s' % (self.line,message) )

    def _beginReadingFile( self, virtual_path ):
        self.is_reading_file = True
        self.file_content = ''
        self.file_path = virtual_path
        self.pending_locations = {}

    def _processReadFile( self, line ):
        cleanLine = line[:]
        while True:
            locIndex = cleanLine.find( '$' )
            if locIndex == -1:
                break
            locEndIndex = cleanLine.find( '$', locIndex+1 )
            if locEndIndex == -1:
                break
            tag = cleanLine[locIndex+1:locEndIndex]
            cleanLine = cleanLine[0:locIndex] + cleanLine[locEndIndex:]
            text_offset = len(self.file_content) + locIndex
            self.pending_locations[tag] = text_offset

        self.file_content += cleanLine            

    # Those should be define in derived class
    def _commentAllowed( self ):
        return True

##    def _doProcessLine( self, line ):
##        pass

    def _doStateChange( self, name, parameter ):
        return False

    def _doProcessReadFile( self, path, content ):
        pass

    def _doTerminateCurrentAction( self ):
        pass

class TestListTestCaseGenerator(LineProcessor):
    """Instiate test case parametrized by a path read from a file, and add them in a test suite."""
    def __init__( self, test_suite, test_class, base_path, list_filename ):
        LineProcessor.__init__( self, None )
        self.base_path = base_path
        self.test_suite = test_suite
        self.test_class = test_class
        test_list_path = os.path.join( base_path, list_filename )
        try:
            self.processFile( test_list_path )
        except LineProcessorError, e:
            class FailingTestCase(unittest.TestCase):
                def __init__( self, message ):
                    unittests.TestCase.__init__( self )
                    self.message = message
                def runTest( self ):
                    raise message
            self.test_suite.addTest( FailingTestCase( "Failed to load test list: " ) )

    def _doProcessLine( self, line ):
        line = line.strip()
        if len(line) > 0:
            self.test_suite.addTest( self.test_class( self.base_path, line ) )
            
class FileSystem:
    def __init__( self ):
        self.next_id = 1
        self.id_by_path = {}
        self.content_by_id = {}
        
    def setFileContent( self, path, content ):
        id = self.getFileId( path )
        self.content_by_id[ id ] = content
        return id

    def getFileId( self, path ):
        if self.id_by_path.has_key( path ):
            return self.id_by_path[ path ]
        id = self.next_id
        self.next_id += 1
        self.id_by_path[ path ] = id
        return id
Index: pycppparser.py
===================================================================
RCS file: /cvsroot/cpptool/CppParser/src/pyrfta/pycppparser.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** pycppparser.py	7 Aug 2004 15:52:24 -0000	1.1
--- pycppparser.py	7 Aug 2004 20:16:06 -0000	1.2
***************
*** 1 ****
--- 1,16 ----
  from pycppparser_ import *
+ 
+ ##for name in dir(NodeSpecificationTypes):
+ ##    if name.startswith( 'NS_' ):
+ ##        print '%s = NodeSpecificationTypes.%s' % (name,name)
+ 
+ # Can't find out how to inject symbol in the current module
+ NS_HAS_BRACE_NODE_VALUE = NodeSpecificationTypes.HAS_BRACE_NODE_VALUE
+ NS_HAS_BRACE_VALUE = NodeSpecificationTypes.HAS_BRACE_VALUE
+ NS_HAS_BRACE_VALUES = NodeSpecificationTypes.HAS_BRACE_VALUES
+ NS_HAS_CHILDREN = NodeSpecificationTypes.HAS_CHILDREN
+ NS_HAS_COLON_ENUMERATION_VALUE = NodeSpecificationTypes.HAS_COLON_ENUMERATION_VALUE
+ NS_HAS_COLON_NODE_VALUE = NodeSpecificationTypes.HAS_COLON_NODE_VALUE
+ NS_HAS_COLON_STRING_LIST_VALUE = NodeSpecificationTypes.HAS_COLON_STRING_LIST_VALUE
+ NS_HAS_COLON_VALUE = NodeSpecificationTypes.HAS_COLON_VALUE
+ NS_NONE = NodeSpecificationTypes.NONE
\ No newline at end of file

Index: pycppparsertest.py
===================================================================
RCS file: /cvsroot/cpptool/CppParser/src/pyrfta/pycppparsertest.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** pycppparsertest.py	7 Aug 2004 15:52:24 -0000	1.1
--- pycppparsertest.py	7 Aug 2004 20:16:06 -0000	1.2
***************
*** 58,62 ****
      def visitChildren( self, node ):
          for child in node.enumChildren():
!             print 'visiting:', child
              self.visitChildren( child )
  
--- 58,62 ----
      def visitChildren( self, node ):
          for child in node.enumChildren():
!             #print 'visiting:', child
              self.visitChildren( child )