[pygccxml-commit] SF.net SVN: pygccxml:[1551] pygccxml_dev

[<rom...@us...>]

Revision: 1551
          http://pygccxml.svn.sourceforge.net/pygccxml/?rev=1551&view=rev
Author:   roman_yakovenko
Date:     2009-01-08 12:31:59 +0000 (Thu, 08 Jan 2009)

Log Message:
-----------
improving support of .so binary file parser

Modified Paths:
--------------
    pygccxml_dev/pygccxml/binary_parsers/__init__.py
    pygccxml_dev/pygccxml/binary_parsers/parsers.py
    pygccxml_dev/pygccxml/binary_parsers/undname.py
    pygccxml_dev/unittests/undname_creator_tester.py

Modified: pygccxml_dev/pygccxml/binary_parsers/__init__.py
===================================================================
--- pygccxml_dev/pygccxml/binary_parsers/__init__.py	2009-01-07 22:25:48 UTC (rev 1550)
+++ pygccxml_dev/pygccxml/binary_parsers/__init__.py	2009-01-08 12:31:59 UTC (rev 1551)
@@ -14,3 +14,7 @@
 def undecorate_blob( blob ):
     """returns undecorated\unmangled string, created from blob"""
     return undname.undname_creator_t().undecorate_blob( blob )
+
+def format_decl( decl, hint=None ):
+    """returns string, that represents formatted decl, according to some rules"""
+    return undname.undname_creator_t().format_decl( decl, hint=hint )

Modified: pygccxml_dev/pygccxml/binary_parsers/parsers.py
===================================================================
--- pygccxml_dev/pygccxml/binary_parsers/parsers.py	2009-01-07 22:25:48 UTC (rev 1550)
+++ pygccxml_dev/pygccxml/binary_parsers/parsers.py	2009-01-08 12:31:59 UTC (rev 1551)
@@ -102,30 +102,32 @@
 
 CCTS = declarations.CALLING_CONVENTION_TYPES
 
-class msvc_libparser_t( libparser_t ):
+class formated_mapping_parser_t( libparser_t ):
     """base parser class for few MSVC binary files"""
-    def __init__( self, global_ns, binary_file ):
+    def __init__( self, global_ns, binary_file, hint ):
         libparser_t.__init__( self, global_ns, binary_file )
         self.__formated_decls = {}
         self.undname_creator = undname.undname_creator_t()
 
+        formatter = lambda decl: self.undname_creator.format_decl( f, hint )
+
         for f in self.global_ns.calldefs( allow_empty=True, recursive=True ):
-            self.__formated_decls[ self.undname_creator.undecorated_decl( f ) ] = f
+            self.__formated_decls[ formatter( f ) ] = f
 
         for v in self.global_ns.variables( allow_empty=True, recursive=True ):
-            self.__formated_decls[ self.undname_creator.undecorated_decl( v ) ] = v
+            self.__formated_decls[ formatter( v ) ] = v
 
     @property
     def formated_decls( self ):
         return self.__formated_decls
 
-class map_file_parser_t( msvc_libparser_t ):
+class map_file_parser_t( formated_mapping_parser_t ):
     """parser for MSVC .map file"""
     c_entry = re.compile( r' +\d+    (?P<internall>.+?)(?:\s+exported name\:\s(?P<name>.*)$)')
     cpp_entry = re.compile( r' +\d+    (?P<decorated>.+?) \((?P<undecorated>.+)\)$' )
 
     def __init__( self, global_ns, map_file_path ):
-        msvc_libparser_t.__init__( self, global_ns, map_file_path )
+        formated_mapping_parser_t.__init__( self, global_ns, map_file_path, 'msvc' )
 
     def load_symbols( self ):
         """returns dictionary { decorated symbol : orignal declaration name }"""
@@ -176,12 +178,12 @@
             return decorated, decl
 
 
-class dll_file_parser_t( msvc_libparser_t ):
+class dll_file_parser_t( formated_mapping_parser_t ):
     """parser for Windows .dll file"""
     def __init__( self, global_ns, map_file_path ):
         global dll_file_parser_warning
         warnings.warn( dll_file_parser_warning, LicenseWarning )
-        msvc_libparser_t.__init__( self, global_ns, map_file_path )
+        formated_mapping_parser_t.__init__( self, global_ns, map_file_path, 'msvc' )
 
     def load_symbols( self ):
         import get_dll_exported_symbols
@@ -207,18 +209,23 @@
 
 
 class so_file_parser_t( libparser_t ):
+    nm_executable = 'nm'
+    #numeric-sort used for mapping between mangled and unmangled name
+    cmd_mangled = '%(nm)s --extern-only --dynamic --defined-only --numeric-sort %(lib)s'
+    cmd_demangled = '%(nm)s --extern-only --dynamic --defined-only --demangle --numeric-sort %(lib)s'
+    entry = re.compile( r'^(?P<address>(?:\w|\d)+)\s\w\s(?P<symbol>.+)$' )
+
     def __init__( self, global_ns, binary_file ):
         libparser_t.__init__( self, global_ns, binary_file )
         self.__mangled2decls = {}
-        
+
         for f in self.global_ns.calldefs( allow_empty=True, recursive=True ):
             self.__mangled2decls[ f.mangled ] = f
-            
+
         for v in self.global_ns.variables( allow_empty=True, recursive=True ):
             self.__mangled2decls[ v.mangled ] = v
-            
-    def load_symbols( self ):
-        cmd = 'nm --extern-only --dynamic --defined-only %s' % self.binary_file
+
+    def __execute_nm( self, cmd ):
         process = subprocess.Popen( args=cmd
                                     , shell=True
                                     , stdin=subprocess.PIPE
@@ -231,26 +238,39 @@
         while process.poll() is None:
             output.append( process.stdout.readline() )
         #the process already finished, read th rest of the output
-        for line in process.stdout.readlines():
-            output.append( line )
+        output.extend( process.stdout.readlines() )
         if process.returncode:
             msg = ["Unable to extract public\\exported symbols from '%s' file." % self.binary_file ]
             msg.append( 'The command line, which was used to extract symbols, is "%s"' % cmd )
             raise RuntimeError( os.linesep.join(msg) )
-            
-        result = []
+        return output
+
+    def __extract_symbols( self, cmd ):
+        output = self.__execute_nm( cmd )
+        result = {}
         for line in output:
-            line = line.strip()
-            if line:
-                result.append( line.split( ' ' )[-1] )
+            found = self.entry.match( line )
+            if found:
+                result[ found.group( 'address' ) ] = found.group( 'symbol' )
         return result
-        
+
+    def load_symbols( self ):
+        tmpl_args = dict( nm=self.nm_executable, lib=self.binary_file )
+        mangled_smbls = self.__extract_symbols( self.cmd_mangled % tmpl_args )
+        demangled_smbls = self.__extract_symbols( self.cmd_demangled % tmpl_args )
+        result = []
+        for address, blob in mangled_smbls.iteritems():
+            if address in demangled_smbls:
+                result.append( blob, demangled_smbls[address] )
+        return result
+
     def merge( self, smbl ):
-        if smbl in self.__mangled2decls:
-            return smbl, self.__mangled2decls[smbl]
-        else:
-            return (None, None)
-            
+        decorated, undecorated = smbl
+        if undecorated not in self.formated_decls:
+            return None, None
+        decl = self.formated_decls[ undecorated ]
+        return decorated, decl
+
 def merge_information( global_ns, fname, runs_under_unittest=False ):
     """high level function - select the appropriate binary file parser and integrates
     the information from the file to the declarations tree. """

Modified: pygccxml_dev/pygccxml/binary_parsers/undname.py
===================================================================
--- pygccxml_dev/pygccxml/binary_parsers/undname.py	2009-01-07 22:25:48 UTC (rev 1550)
+++ pygccxml_dev/pygccxml/binary_parsers/undname.py	2009-01-08 12:31:59 UTC (rev 1551)
@@ -128,7 +128,7 @@
         else:
             return s
 
-    def __format_type_as_undecorated( self, type_, is_argument ):
+    def __format_type_as_undecorated( self, type_, is_argument, hint ):
         result = []
         type_ = declarations.remove_alias( type_ )
         if declarations.is_array( type_ ):
@@ -138,32 +138,42 @@
                 result.append( 'const' )
         else:
             result.append( self.__remove_leading_scope( type_.decl_string ) )
-        return ' '.join( result )
 
+        result = ' '.join( result )
+        if hint == 'nm':
+            for x in ( '*', '&' ):
+                result = result.replace( ' ' + x, x )
+        return result
+
+
     def __normalize( self, name ):
         for what, with_ in self.__fundamental_types:
             name = name.replace( what, with_ )
         name = name.replace( ', ', ',' )
         return name
 
-    def undecorate_argtypes( self, argtypes ):
+    def format_argtypes( self, argtypes, hint ):
         if not argtypes:
-            return 'void'
+            if hint == 'msvc':
+                return 'void'
+            else:
+                return ''
         else:
-            formater = lambda type_: self.__format_type_as_undecorated( type_, True )
+            formater = lambda type_: self.__format_type_as_undecorated( type_, True, hint )
             return ','.join( map( formater, argtypes ) )
 
-    def __undecorated_calldef( self, calldef ):
+    def format_calldef( self, calldef, hint ):
         calldef_type = calldef.function_type()
 
         result = []
         is_mem_fun = isinstance( calldef, declarations.member_calldef_t )
-        if is_mem_fun and calldef.virtuality != declarations.VIRTUALITY_TYPES.NOT_VIRTUAL:
+        if is_mem_fun and hint == 'msvc' and calldef.virtuality != declarations.VIRTUALITY_TYPES.NOT_VIRTUAL:
             result.append( 'virtual ' )
-        if is_mem_fun and calldef.has_static:
+        if is_mem_fun and hint == 'msvc'and calldef.has_static:
             result.append( 'static ' )
-        if calldef_type.return_type:
-            result.append( self.__format_type_as_undecorated( calldef.return_type, False ) )
+        if  hint == 'msvc' and calldef_type.return_type:
+            #nm doesn't dump return type information
+            result.append( self.__format_type_as_undecorated( calldef.return_type, False, hint ) )
             result.append( ' ' )
         if is_mem_fun:
             result.append( self.__remove_leading_scope( calldef.parent.decl_string ) + '::')
@@ -171,35 +181,50 @@
         result.append( calldef.name )
         if isinstance( calldef, ( declarations.constructor_t, declarations.destructor_t) ) \
            and declarations.templates.is_instantiation( calldef.parent.name ):
-            result.append( '<%s>' % ','.join( declarations.templates.args( calldef.parent.name ) ) )
+            if hint == 'msvc':
+                result.append( '<%s>' % ','.join( declarations.templates.args( calldef.parent.name ) ) )
 
-        result.append( '(%s)' % self.undecorate_argtypes( calldef_type.arguments_types ) )
+        result.append( '(%s)' % self.format_argtypes( calldef_type.arguments_types, hint ) )
         if is_mem_fun and calldef.has_const:
-            result.append( 'const' )
+            if hint == 'msvc':
+                result.append( 'const' )
+            else:
+                result.append( ' const' )
         return ''.join( result )
 
-    def __undecorated_variable( self, decl ):
+    def format_var( self, decl, hint ):
         result = []
         is_mem_var = isinstance( decl.parent, declarations.class_t )
         if is_mem_var and decl.type_qualifiers.has_static:
             result.append( 'static ' )
-        result.append( self.__format_type_as_undecorated( decl.type, False ) )
-        result.append( ' ' )
+        if hint == 'msvc':
+            result.append( self.__format_type_as_undecorated( decl.type, False, hint ) )
+            result.append( ' ' )
         if is_mem_var:
             result.append( self.__remove_leading_scope( decl.parent.decl_string ) + '::' )
         result.append( decl.name )
         return ''.join( result )
 
-    def undecorated_decl(self, decl):
+    def format_decl(self, decl, hint=None):
         """returns string, which contains full function name formatted exactly as
         result of dbghelp.UnDecorateSymbolName, with UNDNAME_NO_MS_KEYWORDS | UNDNAME_NO_ACCESS_SPECIFIERS | UNDNAME_NO_ECSU
         options.
+
+        Different compilers\utilities undecorate/demangle magled string ( unique names ) in a different way.
+        hint argument will tell pygccxml how to format declarations, so it couls be mapped later to the blob.
+        The valid options are" msvc, nm
         """
         name = None
+        if hint is None:
+            if 'win32' in sys.platform:
+                hint = 'msvc'
+            else:
+                hint = 'nm'
+
         if isinstance( decl, declarations.calldef_t ):
-            name = self.__undecorated_calldef( decl )
+            name = self.format_calldef( decl, hint )
         elif isinstance( decl, declarations.variable_t ):
-            name = self.__undecorated_variable( decl )
+            name = self.format_var( decl, hint )
         else:
             raise NotImplementedError()
         return self.__normalize( name )

Modified: pygccxml_dev/unittests/undname_creator_tester.py
===================================================================
--- pygccxml_dev/unittests/undname_creator_tester.py	2009-01-07 22:25:48 UTC (rev 1550)
+++ pygccxml_dev/unittests/undname_creator_tester.py	2009-01-08 12:31:59 UTC (rev 1551)
@@ -45,8 +45,6 @@
             tester_t.global_ns = declarations.get_global_namespace( decls )
             tester_t.global_ns.init_optimizer()
 
-            declarations.print_declarations( tester_t.global_ns )
-       
             process = subprocess.Popen( args='scons msvc_compiler=%s' % autoconfig.compiler
                                         , shell=True
                                         , stdin=subprocess.PIPE
@@ -117,7 +115,7 @@
 
     def test_z_compare_parsers( self ):
         if 'win32' not in sys.platform:
-            return 
+            return
         dsymbols, dparser = binary_parsers.merge_information( self.global_ns, self.dll_file, runs_under_unittest=True )
         msymbols, mparser = binary_parsers.merge_information( self.global_ns, self.map_file, runs_under_unittest=True )
 
@@ -127,7 +125,7 @@
         for blob, decl in dsymbols.iteritems():
             if blob not in msymbols:
                 was_error = True
-                print '\n%s could not be found in map file loaded symbols' % binary_parsers.undecorate_blob( blob )
+                print '\n%s could not be found in .map file' % binary_parsers.undecorate_blob( blob )
                 #~ self.failUnless( blob in msymbols, binary_parsers.undecorate_blob( blob ) )
             else:
                 mdecl = msymbols[ blob ]
@@ -138,6 +136,12 @@
         if 'linux2' in sys.platform:
             self.__tester_impl( self.so_file )
 
+    def test_print( self ):
+        symbols, parser = binary_parsers.merge_information( self.global_ns, self.map_file, runs_under_unittest=True )
+        for d in symbols.itervalues():
+            print binary_parsers.format_decl( d, 'nm' )
+
+
 def create_suite():
     suite = unittest.TestSuite()
     suite.addTest( unittest.makeSuite(tester_t))


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.