Menu
▾
▴
cpptool-cvs — Mailing list for CVS commit messages.
You can subscribe to this list here.
| 2002 |
Jan
|
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
(37) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2003 |
Jan
(15) |
Feb
(26) |
Mar
(97) |
Apr
(224) |
May
(226) |
Jun
|
Jul
(3) |
Aug
(22) |
Sep
(48) |
Oct
|
Nov
|
Dec
(38) |
| 2004 |
Jan
(28) |
Feb
|
Mar
(1) |
Apr
|
May
|
Jun
(37) |
Jul
|
Aug
(73) |
Sep
|
Oct
|
Nov
|
Dec
|
|
From: <bl...@us...> - 2003-08-27 21:47:54
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test
In directory sc8-pr-cvs1:/tmp/cvs-serv13262
Modified Files:
cppastdump.py
Log Message:
* updated to use the C++ parser facade
Index: cppastdump.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/cppastdump.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** cppastdump.py 26 Aug 2003 18:00:51 -0000 1.1
--- cppastdump.py 27 Aug 2003 21:47:48 -0000 1.2
***************
*** 1,6 ****
# parser C++ file and dump AST to output
! from rfta.parser import grammar
! from rfta.parser import tokenizer
import glob
import sys
--- 1,5 ----
# parser C++ file and dump AST to output
! from rfta.parser.facade import *
import glob
import sys
***************
*** 8,23 ****
def dumpAST( input_file_name ):
! file_in = file(input_file_name, 'rt')
! text = file_in.read()
! file_in.close()
!
! tokens = tokenizer.CppScanner().tokenize( text )
! scanner = grammar.TokenProvider( tokens, grammar.StructuredTokenStream(), None )
! match = grammar.translation_unit_pi.parse( scanner )
! if not match:
print 'Failed to parse file %s' % input_file_name
! sys.exit(1)
!
! root_node = scanner.getStructuredTokenStream().makeNodeTree()
output_file_name = input_file_name + '.txt'
--- 7,15 ----
def dumpAST( input_file_name ):
! facade = CPPParser()
! text, root_node = facade.parseTextFile( input_file_name )
! if not root_node:
print 'Failed to parse file %s' % input_file_name
! return False
output_file_name = input_file_name + '.txt'
***************
*** 31,34 ****
--- 23,27 ----
file_out.close()
+ return True
|
|
From: <bl...@us...> - 2003-08-27 21:38:42
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser
In directory sc8-pr-cvs1:/tmp/cvs-serv11808
Modified Files:
grammar.py grammartest.py node.py
Added Files:
facade.py
Log Message:
* cleaned up the ast structure
* somewhat uniformised 'declarations' ast structure (parameter, constructor/destructor, function, declaration)
* added a facade for the C++ parser
--- NEW FILE: facade.py ---
import grammar
import tokenizer
class CPPParser:
"""Facade for the C++ Parser."""
def parseText( self, source_text ):
"""Parse the specified text as a C++ File.
Returns the 'translation_unit' AST node (instance of rfta.parser.node.Node.
Returns None if the parsing fails (should never happen)"""
tokens = tokenizer.CppScanner().tokenize( source_text )
scanner = grammar.TokenProvider( tokens, grammar.StructuredTokenStream(), None )
match = grammar.translation_unit_pi.parse( scanner )
if not match:
return None
root_node = scanner.getStructuredTokenStream().makeNodeTree()
return root_node.getChildren()[0]
def parseTextFile( self, path ):
"""Load the specified text file and parse it as a C++ File.
Returns a tuple (text,root_ast_node), where root_ast_node is the 'translation_unit'
AST node (instance of rfta.parser.node.Node, and text is the specified file text.
Returns (text,None) if the parsing fails (should never happen)"""
file_in = file(path, 'rt')
text = file_in.read()
file_in.close()
return ( text, self.parseText( text ) )
Index: grammar.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammar.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** grammar.py 26 Aug 2003 17:53:49 -0000 1.3
--- grammar.py 27 Aug 2003 21:38:38 -0000 1.4
***************
*** 345,349 ****
| direct_abstract_declarator_p )
! opt_parameter_decl_assign_p = optional_f( node_f( 'parameter_default_value',
symbol_f('=') >> assignment_expression_pi ) )
--- 345,349 ----
| direct_abstract_declarator_p )
! opt_parameter_decl_assign_p = optional_f( node_f( 'assign_initializer',
symbol_f('=') >> assignment_expression_pi ) )
***************
*** 355,360 ****
parameter_declaration_p = node_f( 'function_parameter',
! node_f('parameter_type', parameter_type_p)
! >> node_f('parameter_name', declarator_p | abstract_declarator_p)
>> opt_parameter_decl_assign_p )
--- 355,360 ----
parameter_declaration_p = node_f( 'function_parameter',
! node_f('declaration_specifier', parameter_type_p)
! >> node_f('declarator', declarator_p | abstract_declarator_p)
>> opt_parameter_decl_assign_p )
***************
*** 385,395 ****
# removed infinite left recursion
direct_declarator_p = proxy_f( ( declarator_id_p
! | (symbol_f('(') >> declarator_p >> ')') )
>> repeat_f( 0, direct_abstract_declarator_fn_suffix_p
| direct_abstract_declarator_array_suffix_p ) )
# Notes: inversed order to parse pointer to member correctly.
! declarator_p.setParser( (ptr_operator_pi >> declarator_p)
! | direct_declarator_p )
initializer_list_p = list_f( initializer_clause_p, ',' )
--- 385,398 ----
# removed infinite left recursion
direct_declarator_p = proxy_f( ( declarator_id_p
! | node_f('braced_declarator', symbol_f('(') >> declarator_p >> ')') )
>> repeat_f( 0, direct_abstract_declarator_fn_suffix_p
| direct_abstract_declarator_array_suffix_p ) )
# Notes: inversed order to parse pointer to member correctly.
! # simplified => introduced repetition of ptr and discarded alternative
! #declarator_p.setParser( (ptr_operator_pi >> declarator_p)
! # | direct_declarator_p )
! declarator_p.setParser( optional_f( node_f( 'ptr_operator_declarator',
! repeat_f(1, ptr_operator_pi) ) ) >> direct_declarator_p )
initializer_list_p = list_f( initializer_clause_p, ',' )
***************
*** 401,405 ****
| (symbol_f('(') >> expression_list_pi >> ')')
! init_declarator_p = node_f( 'init_declarator', declarator_p >> optional_f( initializer_p ) )
init_declarator_list_p = node_f( 'init_declarators', list_f( init_declarator_p, ',' ) )
--- 404,408 ----
| (symbol_f('(') >> expression_list_pi >> ')')
! init_declarator_p = node_f( 'init_declarator', node_f('declarator', declarator_p) >> optional_f( initializer_p ) )
init_declarator_list_p = node_f( 'init_declarators', list_f( init_declarator_p, ',' ) )
***************
*** 525,529 ****
##decl_specifiers_p = buggy # node_f( 'decl_specifiers', repeat_f( 1, decl_specifier_p ) )
! constructor_type_specifier_pi = optional_f(function_specifier_p)
workaround_type_specifier_pi.setParser(
--- 528,532 ----
##decl_specifiers_p = buggy # node_f( 'decl_specifiers', repeat_f( 1, decl_specifier_p ) )
! constructor_type_specifier_pi = optional_f( node_f('declaration_specifier', function_specifier_p) )
workaround_type_specifier_pi.setParser(
***************
*** 587,592 ****
constructor_declaration_p = constructor_type_specifier_pi >> function_declarator_p
! member_declarator_p = longuest_f( declarator_p >> optional_f(pure_specifier_p),
! declarator_p >> optional_f(constant_initializer_p),
optional_f(id_p) >> ':' >> constant_expression_pi )
--- 590,595 ----
constructor_declaration_p = constructor_type_specifier_pi >> function_declarator_p
! member_declarator_p = longuest_f( node_f('declarator', declarator_p) >> optional_f(pure_specifier_p),
! node_f('declarator', declarator_p) >> optional_f(constant_initializer_p),
optional_f(id_p) >> ':' >> constant_expression_pi )
Index: grammartest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammartest.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** grammartest.py 26 Aug 2003 17:53:50 -0000 1.3
--- grammartest.py 27 Aug 2003 21:38:38 -0000 1.4
***************
*** 487,493 ****
def testDeclaratorParser( self ):
! self.checkParser( '& *var[10]', grammar.declarator_p )
! self.checkParser( '*(fn)(int x,double)', grammar.declarator_p )
! self.checkParser( 'MyClass::instance_', grammar.declarator_p )
def testInitializerParser( self ):
--- 487,506 ----
def testDeclaratorParser( self ):
! self.checkProducedTree( '& *var[10]', grammar.declarator_p,
! TestNode( '',
! TestNode( 'ptr_operator_declarator', '&', '*' ),
! TestNode( 'declarator_id',
! JokerTestNode('unqualified_id') ),
! JokerTestNode( 'abstract_array_declarator' ) ) )
! self.checkProducedNodes( 'MyClass::instance_', grammar.declarator_p,
! TestNode( 'declarator_id', JokerTestNode('qualified_id') ) )
! self.checkParser( '(x)', grammar.declarator_p )
! self.checkProducedTree( '(*fn)(int x,double)', grammar.declarator_p,
! TestNode( '',
! TestNode('braced_declarator',
! '(', TestNode( 'ptr_operator_declarator', '*' ),
! JokerTestNode( 'declarator_id' ),
! ')'),
! JokerTestNode( 'function_parameters' ) ) )
def testInitializerParser( self ):
***************
*** 517,527 ****
'(',
TestNode('function_parameter',
! TestNode('parameter_type', fundamental_type_n('int')),
! TestNode('parameter_name', declarator_id_n( id_n('x')) ) ),
',',
TestNode('function_parameter',
! TestNode('parameter_type', fundamental_type_n('int')),
! TestNode('parameter_name', declarator_id_n( id_n('y')) ),
! TestNode('parameter_default_value', '=',
JokerTestNode('assignment_expression'))),
')' ) )
--- 530,540 ----
'(',
TestNode('function_parameter',
! TestNode('declaration_specifier', fundamental_type_n('int')),
! TestNode('declarator', declarator_id_n( id_n('x')) ) ),
',',
TestNode('function_parameter',
! TestNode('declaration_specifier', fundamental_type_n('int')),
! TestNode('declarator', declarator_id_n( id_n('y')) ),
! TestNode('assign_initializer', '=',
JokerTestNode('assignment_expression'))),
')' ) )
***************
*** 681,685 ****
TestNode( 'init_declarators',
TestNode( 'init_declarator',
! declarator_id_n( id_n('x') ),
TestNode( 'assign_initializer', '=',
JokerTestNode( 'assignment_expression' ) ) ) ),
--- 694,698 ----
TestNode( 'init_declarators',
TestNode( 'init_declarator',
! TestNode( 'declarator', declarator_id_n( id_n('x') ) ),
TestNode( 'assign_initializer', '=',
JokerTestNode( 'assignment_expression' ) ) ) ),
***************
*** 704,707 ****
--- 717,722 ----
self.checkParser( 'void f() {}', grammar.declaration_pi )
self.checkParser( 'extern "C" int x;', grammar.declaration_pi )
+ self.checkParser( 'int (*f)(int x) = count;', grammar.declaration_pi )
+ self.checkParser( 'int (x) = 1234;', grammar.declaration_pi )
***************
*** 787,796 ****
TestNode( 'member_declaration',
TestNode( 'declaration_specifier', fundamental_type_n('void') ),
! declarator_id_n( id_n('initialize') ),
! TestNode( 'function_parameters', '(', ')' ),
';' ),
TestNode( 'member_declaration',
TestNode( 'declaration_specifier', fundamental_type_n('int') ),
! declarator_id_n( id_n('x_') ),
';' ),
TestNode( 'using_declaration', 'using',
--- 802,811 ----
TestNode( 'member_declaration',
TestNode( 'declaration_specifier', fundamental_type_n('void') ),
! TestNode( 'declarator', declarator_id_n( id_n('initialize') ),
! TestNode( 'function_parameters', '(', ')' ) ),
';' ),
TestNode( 'member_declaration',
TestNode( 'declaration_specifier', fundamental_type_n('int') ),
! TestNode( 'declarator', declarator_id_n( id_n('x_') ) ),
';' ),
TestNode( 'using_declaration', 'using',
Index: node.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/node.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** node.py 24 Aug 2003 21:00:36 -0000 1.1
--- node.py 27 Aug 2003 21:38:38 -0000 1.2
***************
*** 27,30 ****
--- 27,48 ----
return len(self.getChildren()) > 0
+ def hasChildOfType( self, type ):
+ return len(self.getChildrenOfType(type)) > 0
+
+ def getChildOfType( self, type ):
+ children = self.getChildrenOfType( type )
+ if children:
+ return children[0]
+ return None
+
+ def getChildrenOfType( self, types ):
+ if type(types) == type(''):
+ types = [ types ]
+ children_of_type = []
+ for child in self.children:
+ if child.getType() in types:
+ children_of_type.append( child )
+ return children_of_type
+
def isLeaf( self ):
return False
|
|
From: <bl...@us...> - 2003-08-27 16:53:57
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser
In directory sc8-pr-cvs1:/tmp/cvs-serv5731/src/pyrfta/test/rfta/parser
Modified Files:
grammar.py grammartest.py
Log Message:
* added error recovery for class member specification
Index: grammar.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammar.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** grammar.py 24 Aug 2003 21:00:36 -0000 1.1
--- grammar.py 26 Aug 2003 08:30:04 -0000 1.2
***************
*** 606,611 ****
access_specification_p = node_f( 'access_specification', access_specifier_p >> ':' )
member_specification_p = node_f('member_specification',
! repeat_f( 1, access_specification_p | member_declaration_p ) )
class_key_pi.setParser( one_of_f('class struct union' ) )
--- 606,613 ----
access_specification_p = node_f( 'access_specification', access_specifier_p >> ':' )
+ member_error_p = node_f( 'member_error', block_error_f() )
+
member_specification_p = node_f('member_specification',
! repeat_f( 1, access_specification_p | member_declaration_p | member_error_p ) )
class_key_pi.setParser( one_of_f('class struct union' ) )
Index: grammartest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammartest.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** grammartest.py 24 Aug 2003 21:00:36 -0000 1.1
--- grammartest.py 26 Aug 2003 08:30:04 -0000 1.2
***************
*** 776,779 ****
--- 776,783 ----
TestNode( 'using_id', id_n('Base'), '::', id_n('visit') ),
';' ) ) )
+ self.checkProducedErrorNodes( "typedef std::map<int,std::string> Dico; Dico dico_;", grammar.member_specification_p,
+ TestNode( 'member_specification',
+ JokerTestNode( 'member_error' ),
+ JokerTestNode( 'member_declaration' ) ) ) # need more detailed check
def testClassSpecifierParser( self ):
|
|
From: <bl...@us...> - 2003-08-27 00:04:28
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser In directory sc8-pr-cvs1:/tmp/cvs-serv437/src/pyrfta/test/rfta/parser Modified Files: __init__.py Log Message: no message Index: __init__.py =================================================================== RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/__init__.py,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** __init__.py 24 Aug 2003 21:00:36 -0000 1.1 --- __init__.py 26 Aug 2003 07:53:48 -0000 1.2 *************** *** 1 **** ! # module \ No newline at end of file --- 1 ---- ! # package \ No newline at end of file |
|
From: <bl...@us...> - 2003-08-26 18:26:21
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser
In directory sc8-pr-cvs1:/tmp/cvs-serv1555/src/pyrfta/test/rfta/parser
Modified Files:
grammar.py grammartest.py
Log Message:
* allowed an extra id before the class name in class definition.
* a few extra nodes added to wrap some expressions.
Index: grammar.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammar.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** grammar.py 26 Aug 2003 08:30:04 -0000 1.2
--- grammar.py 26 Aug 2003 17:53:49 -0000 1.3
***************
*** 93,97 ****
id_expression_pi = longuest_f( qualified_id_p,
! node_f( 'unqualified_id', unqualified_id_pi ) )
# C++ Standard p65
--- 93,97 ----
id_expression_pi = longuest_f( qualified_id_p,
! node_f( 'unqualified_id', unqualified_id_pi ) )
# C++ Standard p65
***************
*** 99,103 ****
| ( symbol_f('this') >> rename_last_cmd( 'this_ref' ) )
| node_f('braced_expr', symbol_f('(') >> expression_pi >> symbol_f(')') )
! | id_expression_pi )
# Notes: missing template in scope case
--- 99,103 ----
| ( symbol_f('this') >> rename_last_cmd( 'this_ref' ) )
| node_f('braced_expr', symbol_f('(') >> expression_pi >> symbol_f(')') )
! | node_f( 'id_expression', id_expression_pi ) )
# Notes: missing template in scope case
***************
*** 582,586 ****
constant_initializer_p = symbol_f('=') >> constant_expression_pi
! pure_specifier_p = symbol_f('=') >> ZeroParser()
# added, somehow the grammar is not parsing this
--- 582,586 ----
constant_initializer_p = symbol_f('=') >> constant_expression_pi
! pure_specifier_p = node_f( 'pure_specifier', symbol_f('=') >> ZeroParser() )
# added, somehow the grammar is not parsing this
***************
*** 625,629 ****
class_head_name_p = node_f( 'class_name',
! optional_f( nested_name_specifier_pi ) >> (id_p | template_id_pi) )
class_head_p = proxy_f( class_key_pi >> optional_f( class_head_name_p ) >> optional_f( base_clause_p ) )
--- 625,631 ----
class_head_name_p = node_f( 'class_name',
! optional_alternative_f( node_f('dll_macro', id_p ),
! optional_f( nested_name_specifier_pi )
! >> (id_p | template_id_pi) ) )
class_head_p = proxy_f( class_key_pi >> optional_f( class_head_name_p ) >> optional_f( base_clause_p ) )
***************
*** 650,654 ****
mem_initializer_id >> '(' >> optional_f( expression_list_pi ) >> ')' )
! ctor_initializer_p.setParser( symbol_f( ':' ) >> list_f( mem_initializer_p, ',' ) )
# ############### A.8.2: Classes, OverLoading ##################
--- 652,657 ----
mem_initializer_id >> '(' >> optional_f( expression_list_pi ) >> ')' )
! ctor_initializer_p.setParser( node_f( 'constructor_initializer',
! symbol_f( ':' ) >> list_f( mem_initializer_p, ',' ) ) )
# ############### A.8.2: Classes, OverLoading ##################
Index: grammartest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/grammartest.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** grammartest.py 26 Aug 2003 08:30:04 -0000 1.2
--- grammartest.py 26 Aug 2003 17:53:50 -0000 1.3
***************
*** 27,30 ****
--- 27,33 ----
return TestNode( 'declarator_id', TestNode( 'unqualified_id', *ids ) )
+ def unqualified_id_expression_n( identifier ):
+ return TestNode( 'id_expression', unqualified_id_n( identifier ) )
+
class GrammarTest(unittest.TestCase,ParserTestHelperMixIn):
***************
*** 88,92 ****
TestNode( 'expression', ('this_ref','this') ),
')' ) )
! self.checkParser( '::my_id', grammar.primary_expression_p )
self.checkParser( 'MyNamespace::my_var', grammar.primary_expression_p )
self.checkParser( '::MyNamespace::MyClass::member_', grammar.primary_expression_p )
--- 91,97 ----
TestNode( 'expression', ('this_ref','this') ),
')' ) )
! self.checkProducedNodes( '::my_id', grammar.primary_expression_p,
! TestNode( 'id_expression',
! TestNode( 'qualified_id', '::', id_n('my_id') ) ) )
self.checkParser( 'MyNamespace::my_var', grammar.primary_expression_p )
self.checkParser( '::MyNamespace::MyClass::member_', grammar.primary_expression_p )
***************
*** 95,111 ****
self.checkProducedNodes( 'xyz', grammar.primary_expression_p,
! TestNode( 'unqualified_id', id_n( 'xyz' ) ) )
self.checkProducedNodes( 'operator =', grammar.primary_expression_p,
! TestNode( 'unqualified_id',
! TestNode( 'operator_function_ref', 'operator', '=' ) ) )
self.checkProducedNodes( 'operator int', grammar.primary_expression_p,
! TestNode( 'unqualified_id',
! TestNode( 'conversion_function_ref',
! 'operator',
! TestNode( 'conversion_type', fundamental_type_specifier_n('int') ) ) ) )
self.checkProducedNodes( '~ClassName()', grammar.primary_expression_p,
! TestNode( 'unqualified_id',
! TestNode( 'destructor_ref', '~', id_n('ClassName') ) ) )
!
def testPostFixExpressionSuffix( self ):
self.checkProducedNodes( '[123]', grammar.array_operator_suffix_p,
--- 100,118 ----
self.checkProducedNodes( 'xyz', grammar.primary_expression_p,
! unqualified_id_expression_n( 'xyz' ) )
self.checkProducedNodes( 'operator =', grammar.primary_expression_p,
! TestNode( 'id_expression',
! TestNode( 'unqualified_id',
! TestNode( 'operator_function_ref', 'operator', '=' ) ) ) )
self.checkProducedNodes( 'operator int', grammar.primary_expression_p,
! TestNode( 'id_expression',
! TestNode( 'unqualified_id',
! TestNode( 'conversion_function_ref',
! 'operator',
! TestNode( 'conversion_type', fundamental_type_specifier_n('int') ) ) ) ) )
self.checkProducedNodes( '~ClassName()', grammar.primary_expression_p,
! TestNode( 'id_expression',
! TestNode( 'unqualified_id',
! TestNode( 'destructor_ref', '~', id_n('ClassName') ) ) ) )
def testPostFixExpressionSuffix( self ):
self.checkProducedNodes( '[123]', grammar.array_operator_suffix_p,
***************
*** 130,134 ****
TestNode( 'array_access',
TestNode( 'array_access',
! JokerTestNode( 'unqualified_id' ),
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ), #[1]
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ) ) #[2]
--- 137,141 ----
TestNode( 'array_access',
TestNode( 'array_access',
! JokerTestNode( 'id_expression' ),
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ), #[1]
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ) ) #[2]
***************
*** 158,162 ****
self.checkParser( 'delete ++p', grammar.delete_expression_p )
self.checkProducedNodes( '::delete [] p', grammar.delete_expression_p,
! TestNode( 'delete_operator', '::', 'delete', '[', ']', unqualified_id_n('p') ) )
self.checkParser( 'delete x.z', grammar.delete_expression_p )
--- 165,170 ----
self.checkParser( 'delete ++p', grammar.delete_expression_p )
self.checkProducedNodes( '::delete [] p', grammar.delete_expression_p,
! TestNode( 'delete_operator', '::', 'delete', '[', ']',
! unqualified_id_expression_n('p') ) )
self.checkParser( 'delete x.z', grammar.delete_expression_p )
***************
*** 178,185 ****
TestNode( 'expression_list',
TestNode( 'assignment_expression',
! unqualified_id_n('__FILE__') ),
',',
TestNode( 'assignment_expression',
! unqualified_id_n('__LINE__') ) ),
')'),
TestNode( 'new_type_id', fundamental_type_n('int') ),
--- 186,193 ----
TestNode( 'expression_list',
TestNode( 'assignment_expression',
! unqualified_id_expression_n('__FILE__') ),
',',
TestNode( 'assignment_expression',
! unqualified_id_expression_n('__LINE__') ) ),
')'),
TestNode( 'new_type_id', fundamental_type_n('int') ),
***************
*** 204,212 ****
TestNode( 'unary_operator',
'++',
! unqualified_id_n('my_variable') ) )
self.checkProducedNodes( '!my_variable', grammar.unary_expression_p,
TestNode( 'unary_operator',
'!',
! unqualified_id_n('my_variable') ) )
self.checkParser( '--my_variable', grammar.unary_expression_p )
self.checkParser( '-my_variable', grammar.unary_expression_p )
--- 212,220 ----
TestNode( 'unary_operator',
'++',
! unqualified_id_expression_n('my_variable') ) )
self.checkProducedNodes( '!my_variable', grammar.unary_expression_p,
TestNode( 'unary_operator',
'!',
! unqualified_id_expression_n('my_variable') ) )
self.checkParser( '--my_variable', grammar.unary_expression_p )
self.checkParser( '-my_variable', grammar.unary_expression_p )
***************
*** 215,219 ****
'sizeof',
TestNode( 'unary_operator', '!',
! unqualified_id_n('my_variable') ) ) )
self.checkProducedNodes( 'sizeof(++my_variable)', grammar.unary_expression_p,
TestNode( 'sizeof_operator',
--- 223,227 ----
'sizeof',
TestNode( 'unary_operator', '!',
! unqualified_id_expression_n('my_variable') ) ) )
self.checkProducedNodes( 'sizeof(++my_variable)', grammar.unary_expression_p,
TestNode( 'sizeof_operator',
***************
*** 223,227 ****
TestNode( 'expression',
TestNode( 'unary_operator', '++',
! unqualified_id_n('my_variable') ) ),
')' ) ) )
self.checkParser( 'new int[10]', grammar.unary_expression_p )
--- 231,235 ----
TestNode( 'expression',
TestNode( 'unary_operator', '++',
! unqualified_id_expression_n('my_variable') ) ),
')' ) ) )
self.checkParser( 'new int[10]', grammar.unary_expression_p )
***************
*** 311,315 ****
def testConstantExpressionParser( self ):
self.checkProducedNodes( 'x', grammar.constant_expression_pi,
! TestNode( 'constant_expression', unqualified_id_n('x') ) )
# Statement
--- 319,323 ----
def testConstantExpressionParser( self ):
self.checkProducedNodes( 'x', grammar.constant_expression_pi,
! TestNode( 'constant_expression', unqualified_id_expression_n('x') ) )
# Statement
***************
*** 699,702 ****
--- 707,714 ----
# Class
+ def testPureSpecifier( self ):
+ self.checkProducedNodes( ' = 0', grammar.pure_specifier_p,
+ TestNode( 'pure_specifier', '=', int_n(0) ) )
+
def testBaseClauseParser( self ):
self.checkParser( ": Parent1", grammar.base_clause_p )
***************
*** 708,711 ****
--- 720,724 ----
self.checkParser( "class MyClass : public Parent", grammar.class_head_p )
self.checkParser( "class MyNamespace::MyClass : public Parent, Parent2Impl", grammar.class_head_p )
+ self.checkParser( 'class EXPORT_MACRO MyClass : public Parent', grammar.class_head_p )
def testMemInitializer( self ):
***************
*** 716,719 ****
--- 729,740 ----
TestNode('mem_initializer_id', '::', id_n('Tools'), '::', id_n('Base') ),
'(', JokerTestNode('expression_list', ), ')' ) )
+
+ def testCtorInitializer( self ):
+ self.checkProducedNodes( ': x_(0), y_(0)', grammar.ctor_initializer_p,
+ TestNode( 'constructor_initializer',
+ ':',
+ JokerTestNode( 'mem_initializer' ),
+ ',',
+ JokerTestNode( 'mem_initializer' ) ) )
def testMemberDeclarationParser( self ):
|
|
From: <bl...@us...> - 2003-08-26 18:01:42
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test
In directory sc8-pr-cvs1:/tmp/cvs-serv2688
Added Files:
cppastdump.py
Log Message:
* a directory is specified in the command instead of a single file.
--- NEW FILE: cppastdump.py ---
# parser C++ file and dump AST to output
from rfta.parser import grammar
from rfta.parser import tokenizer
import glob
import sys
import os.path
def dumpAST( input_file_name ):
file_in = file(input_file_name, 'rt')
text = file_in.read()
file_in.close()
tokens = tokenizer.CppScanner().tokenize( text )
scanner = grammar.TokenProvider( tokens, grammar.StructuredTokenStream(), None )
match = grammar.translation_unit_pi.parse( scanner )
if not match:
print 'Failed to parse file %s' % input_file_name
sys.exit(1)
root_node = scanner.getStructuredTokenStream().makeNodeTree()
output_file_name = input_file_name + '.txt'
file_out = file( output_file_name, 'wt' )
print >>file_out, 'Successfully parsed %s:' % input_file_name
print >>file_out, text
print >>file_out
print >>file_out, 'Produced AST:'
print >>file_out, root_node.getPrettyString()
file_out.close()
if len(sys.argv) != 2:
print 'Usage: cppastdump.py directory'
print 'The program will parse all .cpp and .h files contained in the specified directory.'
print 'AST will be output in the same directory, in a file .txt extension.'
sys.exit(2)
input_directory = sys.argv[1]
filenames = glob.glob(os.path.join(input_directory, '*.cpp' ))
filenames.extend( glob.glob(os.path.join(input_directory, '*.h' )) )
for filename in filenames:
print 'Parsing', filename
dumpAST( filename )
|
|
From: <bl...@us...> - 2003-08-25 21:59:59
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/pytest In directory sc8-pr-cvs1:/tmp/cvs-serv12893/src/pyrfta/pytest Added Files: .cvsignore Log Message: * added parser overview documentation --- NEW FILE: .cvsignore --- *.sln *.suo *.pyc |
|
From: <bl...@us...> - 2003-08-25 21:59:57
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test In directory sc8-pr-cvs1:/tmp/cvs-serv12893/src/pyrfta/test Added Files: .cvsignore Log Message: * added parser overview documentation --- NEW FILE: .cvsignore --- *.sln *.suo *.pyc |
|
From: <bl...@us...> - 2003-08-25 21:59:57
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta In directory sc8-pr-cvs1:/tmp/cvs-serv12893/src/pyrfta/test/rfta Added Files: .cvsignore Log Message: * added parser overview documentation --- NEW FILE: .cvsignore --- *.sln *.suo *.pyc |
|
From: <bl...@us...> - 2003-08-25 21:59:54
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser
In directory sc8-pr-cvs1:/tmp/cvs-serv12893/src/pyrfta/test/rfta/parser
Modified Files:
nodetesthelper.py parsertesthelper.py
Added Files:
.cvsignore readme.txt
Log Message:
* added parser overview documentation
--- NEW FILE: .cvsignore ---
*.pyc
--- NEW FILE: readme.txt ---
1) Overview
The parser implements the full C++ grammar, but does not support template yet.
The grammar is based on the one found in 'The C++ Programming Language, 3rd Edition',
by Bjarne Stroustrup, which is a condensed version of the grammar defined in the C++
98 standard: 'ISO/IEC 14882, Programming Languages - C++'.
The parser framework is what I believe is called a LL(n) parser (recursive descendent parser with infinite look ahead).
The rule of the grammar have the same name where the grammar was left unmodified. This makes it easy to see what are the constraint surrounding a rule. Major to the grammar includes:
- removal of infinite left recursion
- adaptation to a 'I don't known the nature of this identifier' environment (this mainly affected declarator/declaration parsing)
Here is a sketch about the whole parsing process:
- First the Tokenizer transform the text to parser into a sequence of tokens. A token as a type (integer, string, symbol...), its position in the text, as well as the text it represents.
- the token sequence is passed to a TokenProvider. The token provider also takes a StructuredTokenStream and an optional ParsingTracker at construction. The TokenProvider is responsible for providing the 'next token' to use for matching to the parsers. It also implements a transactional semantic: the position can be rollbacked to a previously memorized position (this is how alternative rule are implemented). The StructuredTokenStream stores the output of the parser (what will become an AST later). The ParsingTracker is used for debugging. It is called when each parser is entered and exited (with the matching result).
- Finally, the top-level parser is called with the TokenProvider as a parameter. All parser derives from Parser, which basically as I main method: parse, which takes the TokenProvider as paramter, and returns a boolean indicating if the parser matched the tokens. Here is what the symbol parser (a symbol that match a keyword or an operator) do:
- indicates to the TokenProvider that the parser is entered (which forward the information to the ParsingTracker)
- ask the next token to the TokenProvider
- check if the token is a symbol, and if the token text matched the expected symbol (passed at the construction of the parser)
- if there is a match, the token is pushed to the StructuredTokenStream which is obtained from the TokenProvider
- exit and signal the TokenProvider indicating if there was a match or not.
All parsers follow a similar logic. There is a few key parser, such as the SequenceParser, which just call parse() on a sequence of parser that was passed at the construction, and AlternativeParser, which does a similar thing, but opens a 'transaction' on the TokenProvider before calling parse(), rollback if there was no match, and exit immediately with success if there was a match. There is also a special kind of parser, which always returns a match, and push a 'command' into the StructuredTokenStream. We'll see how this is used later.
When the top-level parser exit, it indicates if it successfully matched the token sequence. The result of the parsing is stored in the StructuredTokenStream.
- The StructuredTokenStream is processed and converted into an AST tree. The StructuredTokenStream is a sequence of matched tokens intermixed with command objects. The command objects indicates how the AST should be created. It is a simple postfix language. When processing the structured token stream after the parsing to create the AST, we starts with a node which is the 'current node'. If a Token is found in the sequence, then it is added to current node. If a command is found, then it is executed. A command can change the 'current node' and do just about anything. Some common commands are 'memorize currrent node', 'restore current node from memory', 'create new node', 'rename current node', 'remove previous node'. There is also some more complex commands like 'apply binary operator node transformation' which make some interesting transformation in the resulting AST... Its simple and allow for complex AST transformations.
When writting with the grammar, the user is shielded from those low level commands by helper factory functions.
2) Playing with the parser
Unit tests are the best way to understand things. The most interesting things to begin with are probably parsertest (parser framework UT)and grammartest (grammar UT).
You can activate the 'debug' mode of the parser to see what are the matched rules and what is the produced AST. This change a little depending on the method used to test the parser. There is two generic cases:
a) the checkParser() methods:
self.checkParser( "int x;", grammar.translation_unit_pi )
If not defined, you need to specify the expect number of token not used by the parser (defaulted to 0) before the debug mode:
self.checkParser( "int x;", grammar.translation_unit_pi, 0, RESULT_DUMP )
b) the checkProducedNodes/Tree() methods:
self.checkProducedNodes( 'operator =', grammar.unqualified_id_pi,
TestNode( 'operator_function_ref', 'operator', '=' ) )
You just need to specify the debug mode:
self.checkProducedNodes( 'operator =', grammar.unqualified_id_pi,
TestNode( 'operator_function_ref', 'operator', '=' ),
MATCHED_TRACE )
At the current time, there is three levels of debug mode (defined in parsertesthelper.py):
MATCHED_TRACE: dump the matched rules 'path' to the output, and the produced AST tree.
TRACE: dump each rule execution to the output, and the produced AST tree.
RESULT_DUMP: only dump the produced AST tree
3) Running the parser
This parser was developped on a old Cyrix 200 (yes that's still exist) running Python 2.3. It as also been tested with Python 2.2.
CD to the top-level package of the parser (src/pyrfta/test), above the
rfta/ and mock/ packages.
Run the alltests.py script ('python alltests.py').
Baptiste Lepilleur, 08/2003.
Index: nodetesthelper.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/nodetesthelper.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** nodetesthelper.py 24 Aug 2003 21:00:36 -0000 1.1
--- nodetesthelper.py 25 Aug 2003 21:48:05 -0000 1.2
***************
*** 36,39 ****
--- 36,47 ----
class TestNode(node.Node):
def __init__( self, name, *children ):
+ """'name' is the type of the expected node.
+ 'children' is a sequence formed from the remaining parameters passed at construction.
+ Each parameter represents a children. The parameter type is tested to allow compact
+ tree specification.
+ None: do nothing
+ 'text': (string) creates and add a TestTokenNode of type SYMBOL with the specified text.
+ (token_type,token_text): (tuple) creates a TestTokenNode
+ TestNode: composite child to add."""
actual_children = []
for child in children:
***************
*** 106,109 ****
--- 114,128 ----
def assertTreeEqual( self, left, right ):
+ """Checks that the 'left' and 'right' node are similar. The left node must only contains
+ instance of TestNode, JokerTestNode and TestTokenNode.
+ The comparison is done by traversing the left and right tree in parallel.
+ A TestNode checks that the left and right node have the same type, the same child count
+ and that their children are similar.
+ A JokerTestNode only checks that the left and right node have the same type. Children are
+ not tested.
+ A TestTokenNode checks that the left and right token node have the same type and text.
+ If a difference is found, the left and right tree are dumped, as well as a diagnostic
+ string pinpointing where the difference is.
+ """
self.assert_( left == right,
'=> Expected tree:\n%s\n=> Actual tree:\n%s\nDifferences:\n%s\n' %
Index: parsertesthelper.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser/parsertesthelper.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** parsertesthelper.py 24 Aug 2003 21:00:36 -0000 1.1
--- parsertesthelper.py 25 Aug 2003 21:48:05 -0000 1.2
***************
*** 31,34 ****
--- 31,40 ----
def checkParser( self, text, tested_parser, expected_remaining_tokens = 0, debug_level = NO_TRACE ):
+ """Checks that 'tested_parser' matches the specified 'text'.
+ 'expected_remaining_tokens' indicates the number of tokens that are expected not to
+ have been consumed by the parser (useful to test that a parser stop at a given token.
+ If the parsing is succesful, the produced AST is explored to ensure that no node
+ containing 'error' in the type/name exist (used for parser error recovery).
+ 'debug_level' indicates the level of debug output."""
match, tracker, scanner = self.doParsing( text, tested_parser, debug_level )
self.assert_( match, "parser should have matched source" )
***************
*** 49,52 ****
--- 55,60 ----
def checkParserFail( self, text, tested_parser, debug_level = NO_TRACE ):
+ """Checks that 'tested_parser' does not match the specified 'text'.
+ 'debug_level' indicates the level of debug output."""
match, tracker, scanner = self.doParsing( text, tested_parser, debug_level )
self.assert_( not match, "parse should not have matched source" )
***************
*** 55,62 ****
--- 63,77 ----
def checkProducedNodes( self, text, tested_parser, expected_tree, debug_level = NO_TRACE, _allow_error_node = False ):
+ """Checks that the 'tested_parser' matches the specified 'text', and that the produced AST matches
+ the 'expected_tree'. See NodeTestHelperMixIn.assertTreeEqual() for detail about the comparison.
+ A TestNode is automatically added at the top of the 'expected_tree' before the comparison.
+ If the parsing is succesful, the produced AST is explored to ensure that no node
+ containing 'error' in the type/name exist (used for parser error recovery).
+ 'debug_level' indicates the level of debug output."""
expected_tree = TestNode( '', expected_tree )
self.checkProducedTree( text, tested_parser, expected_tree, debug_level, _allow_error_node )
def checkProducedTree( self, text, tested_parser, expected_tree, debug_level = NO_TRACE, _allow_error_node = False ):
+ """Same as checkProducedNodes(), but does not add a TestNode at the top of the 'expected_tree'."""
if type(expected_tree) == type( () ):
expected_tree = TestNode( '', expected_tree )
***************
*** 74,80 ****
--- 89,97 ----
def checkProducedErrorNodes( self, text, tested_parser, expected_tree, debug_level = NO_TRACE ):
+ """Same as checkProducedNodes(), but allow the presence of error node in the produced AST."""
self.checkProducedNodes( text, tested_parser, expected_tree, debug_level, True )
def checkProducedErrorTree( self, text, tested_parser, expected_tree, debug_level = NO_TRACE ):
+ """Same as checkProducedTree(), but allow the presence of error node in the produced AST."""
self.checkProducedTree( text, tested_parser, expected_tree, debug_level, True )
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser
In directory sc8-pr-cvs1:/tmp/cvs-serv3499/src/pyrfta/test/rfta/parser
Added Files:
__init__.py alltests.py generateparserdebuginfo.py grammar.py
grammartest.py node.py nodetest.py nodetesthelper.py parser.py
parsertest.py parsertesthelper.py parsingtracker.py
structuredtokenstream.py structuredtokenstreamtest.py token.py
tokenizer.py tokenizertest.py
Log Message:
* added C++ parser in python (without C++ template for the time being), as well as a load of tests.
--- NEW FILE: __init__.py ---
# module
--- NEW FILE: alltests.py ---
#!/usr/bin/env python
#
# Example of assembling all available unit tests into one suite. This usually
# varies greatly from one project to the next, so the code shown below will not
# be incorporated into the 'unittest' module. Instead, modify it for your own
# purposes.
#
# $Id: alltests.py,v 1.1 2003/08/24 21:00:36 blep Exp $
import unittest
import nodetest
import parsertest
import grammartest
import tokenizertest
def suite():
alltests = unittest.TestSuite()
alltests.addTest(unittest.findTestCases(nodetest))
alltests.addTest(unittest.findTestCases(tokenizertest))
alltests.addTest(unittest.findTestCases(parsertest))
alltests.addTest(unittest.findTestCases(grammartest))
# for module in map(__import__, modules_to_test):
# alltests.addTest(unittest.findTestCases(module))
return alltests
if __name__ == '__main__':
unittest.TextTestRunner().run(suite())
--- NEW FILE: generateparserdebuginfo.py ---
# generate parser name for debug
import grammar
import string
for var in dir(grammar):
if var.endswith('_p') or var.endswith('_pi'):
print "%s.setName( '%s' )" % (var,var)
--- NEW FILE: grammar.py ---
# c++ grammar
from parser import *
# IMPORTANT NOTES:
# If a parser rule is stored in a variable and is an Alternative or a Sequence parser, then it must be wrapped
# in a proxy parser, other the parser rule will be extended when used within another rule.
#
# my_p = id_p >> '::' >> id_p # BAD
# my_p = proxy_f( id_p >> '::' >> id_p ) # GOOD
# other_p = my_p >> ',' # in the BAD case, the sequence parser 'my_p' would be extended
# Remarks:
# Because the grammar does use know declared type, it is not possible to distinguish between:
# 1) In the context of an expression:
# function call and object instantiation => node(1) may be either a function call, or a object instantiation,
# or a functional style conversion
# => in this case, the grammar create a 'call_or_conversion_expression'
# 2) In the context of a declaration:
# function declarator and object instantiation => node n(2); may be either a function declaration or an object instantiation.
# => in this case, the grammar create a function_declaration.
id_p = TerminalParser( IDENTIFIER )
eos_p = symbol_f( ';' )
pp_directive_pi = rename_f( 'preprocessor_directive', terminal_f( PP_DIRECTIVE ) )
# ############################## A.4: Program ############################
# A.4, C++PL3 p.798
declaration_seq_pi = proxy_f()
translation_unit_pi = node_f( 'translation_unit', optional_f( declaration_seq_pi ) )
# ############################## A.5: Expressions ############################
# A.5, C++PL3 p.798-802
operator_function_id_pi = proxy_f()
conversion_function_id_pi = proxy_f()
simple_type_specifier_p = proxy_f()
ptr_operator_pi = proxy_f()
type_specifier_seq_pi = proxy_f()
throw_expression_pi = proxy_f()
class_name_pi = proxy_f()
nested_name_specifier_pi = proxy_f()
template_id_pi = proxy_f()
type_id_pi = proxy_f()
expression_pi = proxy_f()
expression_list_pi = proxy_f()
cast_expression_p = proxy_f()
assignment_expression_pi = proxy_f()
unary_expression_p = proxy_f()
constant_expression_pi = proxy_f()
unqualified_id_pi = proxy_f()
new_type_pi = proxy_f() # define in declarators
type_name_pi = proxy_f()
literal_expression_p = proxy_f( ( terminal_f(INTEGER)
| terminal_f(CHAR)
| terminal_f(STRING)
| terminal_f(FLOAT)
| one_of_f( 'true false' ) ) >> rename_last_cmd( 'literal' ) )
namespace_name_pi = proxy_f( id_p )
class_or_namespace_name_p = class_name_pi | namespace_name_pi
template_name_specifier_p = (symbol_f('template') >> nested_name_specifier_pi)
nested_name_specifier_pi.setParser(
repeat_f( 1, class_or_namespace_name_p >> '::' >> optional_f(template_name_specifier_p) ) )
# Notes: qualified_id can not include 'template' => A::template B::zozo is not parsed (IT SHOULD BE)
qualified_id_p = node_f('qualified_id',
( optional_f('::') >> nested_name_specifier_pi >> optional_f('template') >> unqualified_id_pi )
| ( symbol_f('::') >> ( id_p
| operator_function_id_pi
| conversion_function_id_pi ) ) )
# Notes: template id not included
# destructor_ref may be parsed as such while it is not a reference to a destructor (overloaded ~ operator & ())
unqualified_id_pi.setParser( id_p
| operator_function_id_pi
| conversion_function_id_pi \
| node_f('destructor_ref', symbol_f('~') >> id_p >> look_ahead_f( symbol_f('(') >> ')' ) )
) # | template_id_pi
id_expression_pi = longuest_f( qualified_id_p,
node_f( 'unqualified_id', unqualified_id_pi ) )
# C++ Standard p65
primary_expression_p = proxy_f( literal_expression_p
| ( symbol_f('this') >> rename_last_cmd( 'this_ref' ) )
| node_f('braced_expr', symbol_f('(') >> expression_pi >> symbol_f(')') )
| id_expression_pi )
# Notes: missing template in scope case
pseudo_destructor_name_p = node_f( 'pseudo_destructor_name',
optional_f('::') >> optional_f(nested_name_specifier_pi) >> optional_f(type_name_pi >> '::')
>> symbol_f('~') >> id_p )
instance_accessor_p = one_of_f('. ->')
cast_keywords_p = one_of_f('dynamic_cast static_cast reinterpret_cast const_cast')
# Notes: not included in postfix_expression_p:
# typename with template in scope
#
# postfix_expression_p as been modified to avoid the infinite left recursion. This recursion has been
# moved to multiple 'optional' suffixes
#
array_operator_suffix_p = node_f('array_index', symbol_f('[') >> expression_pi >> ']')
call_operator_suffix_p = node_f('call_parameters', symbol_f('(') >> optional_f( expression_list_pi ) >> ')')
accessed_member_p = pseudo_destructor_name_p | (optional_f( 'template' ) >> optional_f( '::' ) >> id_expression_pi)
member_access_suffix_p = node_f('accessed_member', instance_accessor_p >> accessed_member_p )
post_incdec_suffix_p = node_f( 'operator', one_of_f( '++ --' ) )
cppcast_expression_p = node_f('cppcast_expression',
cast_keywords_p >> '<' >> type_id_pi >> '>' >> '(' >> expression_pi >> ')' )
type_id_expression_p = node_f( 'typeid_expression',
symbol_f('typeid') >> '(' >> (type_id_pi | expression_pi) >> ')' )
# Notes: because of the typeless parsing a function call can not be distinguished from a
# functional type conversion. In the case of a function call, the name of the function will be found in the
# type id.
call_or_conversion_expression_p = node_f( 'call_or_conversion_expression',
( (symbol_f('typename') >> optional_f('::') >> nested_name_specifier_pi >> id_p)
| simple_type_specifier_p )
>> '(' >> optional_f( expression_list_pi ) >> ')' )
#Notes: template_id construction is missing
postfix_expression_p = binary_operator_f(
alternative_f( cppcast_expression_p,
type_id_expression_p,
call_or_conversion_expression_p,
primary_expression_p ),
alternative_f( make_right_op_f(array_operator_suffix_p, 'array_access'), # longuest could be removed if instance_accessor stuff are factorized into one rule
make_right_op_f(call_operator_suffix_p, 'call'),
make_right_op_f(member_access_suffix_p, 'member_access'),
make_right_op_f(post_incdec_suffix_p, 'post_incdec') ) )
cast_expression_p = proxy_f()
cast_expression_p.setParser( unary_expression_p
| node_f('cast_expression',
symbol_f('(') >> type_id_pi >> ')' >> node_f('casted_expression',
cast_expression_p ) ) )
delete_expression_p = node_f( 'delete_operator',
optional_f( '::') >> 'delete' >> optional_f(symbol_f('[') >> ']') >> cast_expression_p)
new_initializer_p = node_f('new_initializer', symbol_f('(') >> optional_f( expression_list_pi ) >> ')')
expression_list_pi.setParser( node_f( 'expression_list', list_f( assignment_expression_pi, ',' ) ) )
direct_new_declarator_p = repeat_f( 1, symbol_f('[') >> expression_pi >> ']' ) \
>> optional_f( symbol_f('[') >> constant_expression_pi >> ']' )
new_declarator_p = proxy_f()
new_declarator_p.setParser( (ptr_operator_pi >> optional_f( new_declarator_p ))
| direct_new_declarator_p )
new_type_id_p = node_f('new_type_id', new_type_pi >> optional_f( new_declarator_p ))
new_type_id_alt_p = (symbol_f('(') >> new_type_id_p >> ')') \
| new_type_id_p
new_placement_p = node_f('new_placement', symbol_f( '(' ) >> expression_list_pi >> ')')
new_expression_p = node_f('new_operator',
optional_f( '::' ) >> 'new' >>
optional_alternative_f( new_placement_p,
new_type_id_alt_p >> optional_f(new_initializer_p) ))
# Notes: added '++' & '--'
unary_operator_p = one_of_f( '++ -- * & + - ! ~' )
# Notes: sizeof( expr ) is already handled because unary_expression may be a braced expression.
unary_expression_p.setParser( postfix_expression_p
| node_f('unary_operator', unary_operator_p >> cast_expression_p)
| node_f('sizeof_operator', symbol_f( 'sizeof' ) >> unary_expression_p)
| new_expression_p
| delete_expression_p )
pm_expression_p = binary_operator_f( cast_expression_p,
make_right_op_f( node_f('right_operand',
instance_accessor_p >> '*' >> cast_expression_p ),
'pointer_to_member_operator' ) )
##def binary_expr_p_factory( simple_expr_parser, symbols ):
## return proxy_f( simple_expr_parser >> repeat_f( 0, one_of_f( symbols ) >> simple_expr_parser ) )
def binary_expr_p_factory( simple_expr_parser, symbols ):
return binary_operator_f( simple_expr_parser,
make_right_op_f( node_f('right_operand',
one_of_f( symbols ) >> simple_expr_parser),
'binary_operator' ) )
multiplicative_expression_p = binary_expr_p_factory( pm_expression_p, '* / %' )
additive_expression_p = binary_expr_p_factory( multiplicative_expression_p, '+ -' )
shift_expression_p = binary_expr_p_factory( additive_expression_p, '<< >>' )
relational_expression_p = binary_expr_p_factory( shift_expression_p, '< > <= >=' )
equality_expression_p = binary_expr_p_factory( relational_expression_p, '== !=' )
and_expression_p = binary_expr_p_factory( equality_expression_p, '&' )
exclusive_or_expression_p = binary_expr_p_factory( and_expression_p, '^' )
inclusive_or_expression_p = binary_expr_p_factory( exclusive_or_expression_p, '|' )
logical_and_expression_p = binary_expr_p_factory( inclusive_or_expression_p, '&&' )
logical_or_expression_p = binary_expr_p_factory( logical_and_expression_p, '||' )
##conditional_expression_p = proxy_f( logical_or_expression_p
## >> repeat_f( 0, symbol_f('?') >> expression_pi >> ':' >> assignment_expression_pi ) )
conditional_expression_p = binary_operator_f(
logical_or_expression_p,
make_right_op_f( node_f('right_operand', symbol_f('?') >> expression_pi >> ':' >> assignment_expression_pi ),
'tertiary_operator' ) )
assignment_operator_p = one_of_f( '= *= /= %= += -= >>= <<= &= ^= |=' )
assignment_expression_p = longuest_f(
conditional_expression_p,
(logical_or_expression_p >> assignment_operator_p >> assignment_expression_pi),
throw_expression_pi )
expression_pi.setParser( node_f( 'expression',
node_f( 'assignment_expression_list', list_f( assignment_expression_p, ',', 2 ) )
| assignment_expression_p ) )
assignment_expression_pi.setParser( node_f( 'assignment_expression', assignment_expression_p ) )
constant_expression_pi.setParser( node_f( 'constant_expression', conditional_expression_p ) )
# ############################## A.6: Statements ############################
# A.6, C++PL3 p.802-803
block_declaration_p = proxy_f()
declarator_p = proxy_f()
simple_declaration_p = proxy_f()
statement_p = proxy_f()
try_block_pi = proxy_f()
condition_p = expression_pi \
| ( type_specifier_seq_pi >> declarator_p >> '=' >> assignment_expression_pi )
expression_statement_p = node_f( 'expression_statement', optional_f(expression_pi) >> eos_p )
jump_statement_p = node_f( 'break_statement', symbol_f('break') >> eos_p ) \
| node_f( 'continue_statement', symbol_f('continue') >> eos_p ) \
| node_f( 'return_statement', symbol_f('return') >> optional_f(expression_pi) >> eos_p) \
| node_f( 'goto_statement', symbol_f('goto') >> id_p >> eos_p)
for_init_statement_p = expression_statement_p \
| simple_declaration_p
iteration_statement_p = node_f('while_statement',
symbol_f('while') >> '(' >> condition_p >> ')' >> statement_p) \
| node_f('dowhile_statement',
symbol_f('do') >> statement_p >> 'while' >> '(' >> expression_pi >> ')' >> eos_p) \
| node_f('for_statement',
symbol_f('for') >> '(' >> for_init_statement_p
>> node_f('for_condition', optional_f( condition_p ) )
>> eos_p >> optional_f( expression_pi ) >> ')' >> statement_p )
selection_statement_p = node_f( 'if_statement', symbol_f('if') >> '(' >> condition_p >> ')' >> statement_p \
>> optional_f( symbol_f('else') >> statement_p ) ) \
| node_f( 'switch_statement', symbol_f('switch') >> '(' >> condition_p >> ')' >> statement_p )
error_statement_p = node_f( 'error_statement', block_error_f() )
compound_statement_pi = node_f( 'compound_statement',
symbol_f('{') >> repeat_f( 0,
statement_p | error_statement_p ) >> '}' )
labeled_statement_p = node_f('label_statement', id_p >> ':' >> statement_p) \
| node_f( 'case_statement', symbol_f('case') >> constant_expression_pi >> ':' >> statement_p ) \
| node_f( 'default_statement', symbol_f('default') >> ':' >> statement_p )
declaration_statement_p = node_f( 'declaration_statement', block_declaration_p )
statement_p.setParser( labeled_statement_p
| compound_statement_pi
| selection_statement_p
| iteration_statement_p
| jump_statement_p
| expression_statement_p
| declaration_statement_p
| try_block_pi
| pp_directive_pi )
# ############################## A.7.1: Declarators ############################
# A.7.1, C++PL3 p.807-808
class_key_pi = proxy_f()
class_specifier_pi = proxy_f()
ctor_initializer_p = proxy_f()
exception_specification_pi = proxy_f()
function_try_block_pi = proxy_f()
abstract_declarator_p = proxy_f()
direct_abstract_declarator_p = proxy_f()
parameter_declaration_clause_p = proxy_f()
initializer_clause_p = proxy_f()
workaround_type_specifier_pi = proxy_f()
cv_qualifier_p = one_of_f('const volatile' )
cv_qualifier_seq_p = node_f( 'cv_qualifiers', repeat_f( 1, cv_qualifier_p ) )
# buggy standard C++ rules. Use 'custom type parser instead' => see decl_specifiers_p, parameter_type_p,
# new_type_pi, function_return_type...
# decl_specifier_seq_p = buggy # repeat_f( 1, decl_specifier_p )
# decl_specifier_p = buggy #( storage_class_specifier_p
# | type_specifier_p
# | function_specifier_p
# | rename_f( 'friend_specifier', 'friend' )
# | rename_f( 'typedef_specifier', 'typedef' ) )
# type_specifier_p = buggy #( simple_type_specifier_p
# | enum_specifier_p
# | longuest_f( class_specifier_pi, elaborated_type_specifier_p )
# | cv_qualifier_p)
# removed left recursion
direct_abstract_declarator_fn_suffix_p = proxy_f(
node_f( 'function_parameters', symbol_f('(') >> parameter_declaration_clause_p >> ')' )
>> optional_f( cv_qualifier_seq_p )
>> optional_f( exception_specification_pi ) )
direct_abstract_declarator_array_suffix_p = node_f( 'abstract_array_declarator',
symbol_f('[') >> optional_f( constant_expression_pi ) >> ']' )
direct_abstract_declarator_p.setParser(
optional_f( symbol_f('(') >> abstract_declarator_p >> ')' )
>> repeat_f( 0, direct_abstract_declarator_fn_suffix_p
| direct_abstract_declarator_array_suffix_p ) )
abstract_declarator_p.setParser( (ptr_operator_pi >> optional_f( abstract_declarator_p )) \
| direct_abstract_declarator_p )
opt_parameter_decl_assign_p = optional_f( node_f( 'parameter_default_value',
symbol_f('=') >> assignment_expression_pi ) )
parameter_type_p = proxy_f( optional_f(cv_qualifier_seq_p)
>> workaround_type_specifier_pi
>> optional_f(cv_qualifier_seq_p) )
new_type_pi.setParser( workaround_type_specifier_pi )
parameter_declaration_p = node_f( 'function_parameter',
node_f('parameter_type', parameter_type_p)
>> node_f('parameter_name', declarator_p | abstract_declarator_p)
>> opt_parameter_decl_assign_p )
ellipsis_parameter_declaration = node_f('ellipsis_parameter', '...' )
parameter_declaration_clause_p.setParser( ellipsis_parameter_declaration \
| (optional_f( list_f( parameter_declaration_p, ',' ) ) \
>> optional_f( symbol_f(',') >> ellipsis_parameter_declaration ) ) )
enum_name_p = id_p
typedef_name_p = id_p # p793
type_name_pi.setParser( class_name_pi | enum_name_p | typedef_name_p )
# Notes: simplified, alternatives are redondant in typeless context
declarator_id_p = node_f( 'declarator_id', optional_f( '::' ) >> id_expression_pi )
ptr_operator_pi.setParser( (symbol_f('*') >> optional_f( cv_qualifier_seq_p )) \
| symbol_f('&') \
| node_f('pointer_to_member',
optional_f('::') >> nested_name_specifier_pi >> '*' >> optional_f( cv_qualifier_seq_p ) ) )
# defined in A71, declarators, p807 , after cv_qualifier
type_id_pi.setParser( node_f( 'type_id',
type_specifier_seq_pi >> optional_f( abstract_declarator_p ) ) )
# removed infinite left recursion
direct_declarator_p = proxy_f( ( declarator_id_p
| (symbol_f('(') >> declarator_p >> ')') )
>> repeat_f( 0, direct_abstract_declarator_fn_suffix_p
| direct_abstract_declarator_array_suffix_p ) )
# Notes: inversed order to parse pointer to member correctly.
declarator_p.setParser( (ptr_operator_pi >> declarator_p)
| direct_declarator_p )
initializer_list_p = list_f( initializer_clause_p, ',' )
initializer_clause_p.setParser( assignment_expression_pi \
| (symbol_f('{') >> optional_f( initializer_list_p >> optional_f( ',' ) ) >> '}') )
initializer_p = node_f('assign_initializer', symbol_f('=') >> initializer_clause_p ) \
| (symbol_f('(') >> expression_list_pi >> ')')
init_declarator_p = node_f( 'init_declarator', declarator_p >> optional_f( initializer_p ) )
init_declarator_list_p = node_f( 'init_declarators', list_f( init_declarator_p, ',' ) )
type_specifier_seq_pi.setParser( node_f( 'type_specifier',
optional_f(cv_qualifier_seq_p)
>> workaround_type_specifier_pi
>> optional_f(cv_qualifier_seq_p) ) )
function_body_p = compound_statement_pi
#Note: C++ standard, p140:
# a declarator_p in a function definition shall have the form:
# D1 ( parameter_declaration_clause ) optional_f(cv_qualifier_seq) optional_f(exception_specification)
# function_declarator_p implement this instead of using a standard declarator_p.
function_return_type_p = proxy_f( optional_f(cv_qualifier_seq_p)
>> workaround_type_specifier_pi
>> optional_f(cv_qualifier_seq_p) )
function_declarator_p = declarator_id_p >> direct_abstract_declarator_fn_suffix_p
function_definition_start_p = optional_alternative_f( node_f( 'function_return_type', function_return_type_p ),
function_declarator_p )
function_definition_pi = node_f('function_definition',
function_definition_start_p
>> ( (optional_f(ctor_initializer_p) >> function_body_p)
| function_try_block_pi ) )
# ############################## A.7: Declaration ############################
declaration_pi = proxy_f() # forward declaration for recursion
declaration_error_p = node_f( 'declaration_error', block_error_f() )
declaration_seq_pi.setParser( repeat_f( 1, declaration_pi | declaration_error_p ) )
# A.7, C++PL3 p.805-806
# => parse namespace declaration and namespace alias declaration
# Notes: OK. No parsing issue
namespace_body_p = optional_f( node_f( 'namespace_body', declaration_seq_pi ) )
qualified_namespace_specifier_p = node_f( 'namespace_specifier',
optional_f('::') >> optional_f(nested_name_specifier_pi) >> namespace_name_pi )
named_namespace_definition_p = symbol_f('namespace') >> id_p >> '{' >> namespace_body_p >> '}'
unnamed_namespace_definition_p = (symbol_f('namespace') >> '{' >> namespace_body_p >> '}')
namespace_definition_p = node_f( 'named_namespace_def', named_namespace_definition_p) \
| node_f( 'unnamed_namespace_def', unnamed_namespace_definition_p )
namespace_alias_definition_p = node_f('namespace_alias_def',
symbol_f('namespace') >> id_p >> '='
>> qualified_namespace_specifier_p >> eos_p)
# A.7, C++PL3 p.806
# => parse using declaration and directive
# Notes: OK. No parsing issue
using_id_p = node_f( 'using_id',
(optional_f('typename') >> optional_f('::') >> nested_name_specifier_pi >> unqualified_id_pi)
| ( symbol_f('::') >> unqualified_id_pi ) )
using_declaration_p = node_f('using_declaration',
symbol_f('using') >> using_id_p >> eos_p )
using_directive_p = symbol_f('using') >> 'namespace' >> qualified_namespace_specifier_p >> eos_p
# A.7, C++PL3 p.806
# => parse asm definition
asm_keyword_p = one_of_f('asm __asm') # (__asm) => Microsoft Visual C++
asm_definition_p = asm_keyword_p >> '(' >> terminal_f(STRING) >> ')' >> eos_p
# A.7, C++PL3 p.806
# => parse extern specifiction
linkage_specification_begin_p = symbol_f('extern') >> terminal_f(STRING)
linkage_specification_p = node_f( 'linkage_specification',
linkage_specification_begin_p
>> ( (symbol_f('{') >> optional_f(declaration_seq_pi) >> '}')
| declaration_pi ) )
# A.7, C++PL3 p.804-806
# => parse simple declaration
# Notes: parsing issue in:
# -simple_type_specifier_p
# -enumerator_definition_p (for constant expression that contains template with multiple parameters)
enumerator_p = proxy_f( id_p )
enumerator_definition_p = node_f( 'enumerator_definition',
enumerator_p >> optional_f( symbol_f('=') >> constant_expression_pi ) )
enum_specifier_p = node_f( 'enum_specifier',
symbol_f('enum') >> optional_f(id_p) >> '{' >>
optional_f(list_f(enumerator_definition_p, ',') )
>> '}' )
forwarded_type_id_p = node_f( 'forwarded_type_id',
optional_f( '::' ) >> optional_f( nested_name_specifier_pi ) >> id_p )
forward_typename_id_p = node_f( 'forwarded_type_id',
optional_f( '::' ) >> optional_f( nested_name_specifier_pi )
>> ( (symbol_f('template') >> template_id_pi ) | id_p ) )
elaborated_type_specifier_p = proxy_f(
node_f( 'forward_class_specifier', class_key_pi >> forwarded_type_id_p )
| node_f( 'forward_enum_specifier', symbol_f('enum') >> forwarded_type_id_p >> look_ahead_f( not_f( '(' ) ) )
| node_f( 'forward_typename_specifier', symbol_f('typename') >> forward_typename_id_p >> look_ahead_f( not_f( '(' ) ) ) )
fundamental_type = repeat_f( 1,
one_of_f( 'char wchar_t bool short int long signed unsigned float double void' ) )
# Notes: possible optimization issue for type_name => may need longuest_f
user_type_specifier_p = proxy_f(
node_f( 'type_name', optional_f('::') >> nested_name_specifier_pi >> optional_f('template') >> template_id_pi )
| node_f( 'type_name', optional_f('::') >> optional_f(nested_name_specifier_pi) >> type_name_pi ) )
simple_type_specifier_p.setParser(
node_f( 'fundamental_type_specifier', fundamental_type )
| user_type_specifier_p )
storage_class_specifier_p = rename_f('storage_class_specifier',
one_of_f('auto register static extern mutable') )
function_specifier_p = rename_f('function_specifier', one_of_f('inline virtual explicit') )
# Hand made rules concerning the order of decl_specifier
# Those rules allow us to detect that the type specifier of 'MyClass instance' is 'MyClass'
# and not 'MyClass insance'.
# typedef cv_qualifiers type cv_qualifiers
# storage_class_specifier(1) function_specifiers cv_qualifiers type cv_qualifiers
# friend 'storage decl'
# type = repeat(1,base_type) | user_type
##decl_specifiers_p = buggy # node_f( 'decl_specifiers', repeat_f( 1, decl_specifier_p ) )
constructor_type_specifier_pi = optional_f(function_specifier_p)
workaround_type_specifier_pi.setParser(
simple_type_specifier_p
| enum_specifier_p
| longuest_f( class_specifier_pi, elaborated_type_specifier_p ) )
standard_decl_specifier_impl_p = proxy_f(
optional_f( storage_class_specifier_p )
>> optional_f( function_specifier_p )
>> optional_f( cv_qualifier_seq_p )
>> workaround_type_specifier_pi
>> optional_f( cv_qualifier_seq_p ) )
standard_decl_specifiers_p = node_f( 'declaration_specifier', standard_decl_specifier_impl_p )
typedef_decl_specifiers_p = node_f( 'typedef_specifier',
symbol_f('typedef')
>> optional_f(cv_qualifier_seq_p)
>> workaround_type_specifier_pi
>> optional_f(cv_qualifier_seq_p) )
friend_decl_specifiers_p = node_f( 'friend_specifier',
symbol_f('friend') >> standard_decl_specifier_impl_p )
decl_specifiers_p = proxy_f( typedef_decl_specifiers_p
| friend_decl_specifiers_p
| standard_decl_specifiers_p )
simple_declaration_p.setParser( node_f( 'simple_declaration',
optional_f( decl_specifiers_p )
>> optional_f( init_declarator_list_p ) >> eos_p ) )
# A.7, C++PL3 p.804
block_declaration_p.setParser( asm_definition_p
| namespace_alias_definition_p
| using_declaration_p
| using_directive_p
| simple_declaration_p )
declaration_pi.setParser( block_declaration_p
| linkage_specification_p
| namespace_definition_p
| function_definition_pi
| pp_directive_pi )
## | template_declaration_p
## | explicit_instantiation_p
## | explicit_specialization_p )
# ############################## A.8: Classes ############################
# A.8, C++PL3 p.808-809
class_name_pi.setParser( id_p )
constant_initializer_p = symbol_f('=') >> constant_expression_pi
pure_specifier_p = symbol_f('=') >> ZeroParser()
# added, somehow the grammar is not parsing this
constructor_declaration_p = constructor_type_specifier_pi >> function_declarator_p
member_declarator_p = longuest_f( declarator_p >> optional_f(pure_specifier_p),
declarator_p >> optional_f(constant_initializer_p),
optional_f(id_p) >> ':' >> constant_expression_pi )
member_declarator_list_p = list_f( member_declarator_p, ',' )
#Notes: not included:
#- opt(::) nested-name-specifier opt(template) unqualified-id
member_declaration_p = proxy_f(
node_f( 'member_function_definition', (function_definition_pi >> optional_f( eos_p )) )
| node_f('member_declaration', optional_f( decl_specifiers_p ) >> optional_f( member_declarator_list_p ) >> eos_p)
| node_f('typeless_function_declaration', constructor_declaration_p >> eos_p)
| using_declaration_p )
# | template_declaration_p
access_specifier_p = one_of_f( "public protected private" )
access_specification_p = node_f( 'access_specification', access_specifier_p >> ':' )
member_specification_p = node_f('member_specification',
repeat_f( 1, access_specification_p | member_declaration_p ) )
class_key_pi.setParser( one_of_f('class struct union' ) )
base_class_name_p = node_f( 'base_class_name',
optional_f('::') >> optional_f(nested_name_specifier_pi) >> class_name_pi )
base_specifier_p = node_f( 'base_class_access',
optional_f('virtual') >> optional_f( access_specifier_p ) >> optional_f('virtual') ) \
>> base_class_name_p
# Notes: could use a subparser here: match ':' - '{'
base_clause_p = node_f( 'class_bases',
symbol_f(':') >> list_f( node_f( 'base_specifier', base_specifier_p ), ',' ) )
class_head_name_p = node_f( 'class_name',
optional_f( nested_name_specifier_pi ) >> (id_p | template_id_pi) )
class_head_p = proxy_f( class_key_pi >> optional_f( class_head_name_p ) >> optional_f( base_clause_p ) )
class_specifier_pi.setParser( node_f( 'class_specifier',
class_head_p >> '{' >> optional_f( member_specification_p ) >> '}' ) )
# ############### A.8.2: Classes, Specials Member Functions ##################
# A.8.2, C++PL3 p.810
conversion_declarator_p = repeat_f( 1, ptr_operator_pi )
conversion_type_id_p = node_f('conversion_type',
type_specifier_seq_pi >> optional_f( conversion_declarator_p ))
conversion_function_id_pi.setParser( node_f('conversion_function_ref',
symbol_f('operator') >> conversion_type_id_p) )
mem_initializer_id = node_f( 'mem_initializer_id',
(optional_f('::') >> optional_f(nested_name_specifier_pi) >> class_name_pi)
| id_p )
mem_initializer_p = node_f( 'mem_initializer',
mem_initializer_id >> '(' >> optional_f( expression_list_pi ) >> ')' )
ctor_initializer_p.setParser( symbol_f( ':' ) >> list_f( mem_initializer_p, ',' ) )
# ############### A.8.2: Classes, OverLoading ##################
# A.8.3, C++PL3 p.810
operator_p = one_of_f( '+ - * / % ^ & | ~ ! = < > += -= *= /= %= ^= &= |= << >> >>= <<= == != <= >= && || ++ -- , ->* -> () []') \
| ( one_of_f( 'new delete' ) >> optional_f( symbol_f('[') >> ']' ) )
operator_function_id_pi.setParser( node_f('operator_function_ref',
symbol_f('operator') >> operator_p) )
# ############################## A.9: Templates ############################
# A.9, C++PL3 p.811
template_declaration_p = proxy_f()
template_parameter_list_p = proxy_f()
template_name_p = proxy_f( id_p )
template_argument_p = assignment_expression_pi | type_id_pi | template_name_p
template_argument_list_p = list_f( template_argument_p, ',' )
template_id_pi.setParser( template_name_p >> '<' >> optional_f( template_argument_list_p ) >> '>' )
type_parameter_p = proxy_f(
one_of_f('class typename') >> optional_f( id_p ) >> optional_f( symbol_f('=') >> type_id_pi )
| ( symbol_f('template') >> '<' >> template_parameter_list_p >> '>' >> 'class' >> optional_f(id_p)
>> optional_f( symbol_f('=') >> template_name_p ) ) )
template_parameter_p = type_parameter_p | parameter_declaration_p
template_parameter_list_p.setParser( list_f( template_parameter_p, ',' ) )
template_declaration_p.setParser( symbol_f('template') >> '<' >> template_parameter_list_p >> '>' >> declaration_pi )
explicit_instantiation_p = proxy_f( symbol_f('template') >> declaration_pi )
explicit_specialization_p = proxy_f( symbol_f('template') >> '<' >> '>' >> declaration_pi )
# ############################## A.10: Exceptions ############################
# A.10, C++PL3 p.812
throw_expression_pi.setParser( symbol_f('throw') >> optional_f( assignment_expression_pi ) )
exception_declaration_p = (type_specifier_seq_pi >> optional_f( declarator_p | abstract_declarator_p )) \
| '...'
handler_p = node_f('exception_handler',
symbol_f('catch') >> '(' >> exception_declaration_p >> ')' >> compound_statement_pi )
handler_seq_p = repeat_f( 1, handler_p )
type_id_list_p = node_f( 'type_id_list', list_f( type_id_pi, ',' ) )
exception_specification_pi.setParser(
node_f('exception_specification',
symbol_f( 'throw' ) >> '(' >> optional_f( type_id_list_p ) >> ')' ) )
try_block_pi.setParser( node_f( 'try_block', symbol_f('try') >> compound_statement_pi >> handler_seq_p ) )
function_try_block_pi.setParser( symbol_f('try') >> optional_f(ctor_initializer_p) >> function_body_p >> handler_seq_p )
# #################################### Debug information for parser ###################
# generated using generateparserdebuginfo.py (Notes: could this be done dynamically ?)
abstract_declarator_p.setName( 'abstract_declarator_p' )
access_specification_p.setName( 'access_specification_p' )
access_specifier_p.setName( 'access_specifier_p' )
accessed_member_p.setName( 'accessed_member_p' )
additive_expression_p.setName( 'additive_expression_p' )
and_expression_p.setName( 'and_expression_p' )
array_operator_suffix_p.setName( 'array_operator_suffix_p' )
asm_definition_p.setName( 'asm_definition_p' )
asm_keyword_p.setName( 'asm_keyword_p' )
assignment_expression_p.setName( 'assignment_expression_p' )
assignment_expression_pi.setName( 'assignment_expression_pi' )
assignment_operator_p.setName( 'assignment_operator_p' )
base_class_name_p.setName( 'base_class_name_p' )
base_clause_p.setName( 'base_clause_p' )
base_specifier_p.setName( 'base_specifier_p' )
block_declaration_p.setName( 'block_declaration_p' )
call_operator_suffix_p.setName( 'call_operator_suffix_p' )
call_or_conversion_expression_p.setName( 'call_or_conversion_expression_p' )
cast_expression_p.setName( 'cast_expression_p' )
cast_keywords_p.setName( 'cast_keywords_p' )
class_head_name_p.setName( 'class_head_name_p' )
class_head_p.setName( 'class_head_p' )
class_key_pi.setName( 'class_key_pi' )
class_name_pi.setName( 'class_name_pi' )
class_or_namespace_name_p.setName( 'class_or_namespace_name_p' )
class_specifier_pi.setName( 'class_specifier_pi' )
compound_statement_pi.setName( 'compound_statement_pi' )
condition_p.setName( 'condition_p' )
conditional_expression_p.setName( 'conditional_expression_p' )
constant_expression_pi.setName( 'constant_expression_pi' )
constant_initializer_p.setName( 'constant_initializer_p' )
constructor_declaration_p.setName( 'constructor_declaration_p' )
constructor_type_specifier_pi.setName( 'constructor_type_specifier_pi' )
conversion_declarator_p.setName( 'conversion_declarator_p' )
conversion_function_id_pi.setName( 'conversion_function_id_pi' )
conversion_type_id_p.setName( 'conversion_type_id_p' )
cppcast_expression_p.setName( 'cppcast_expression_p' )
ctor_initializer_p.setName( 'ctor_initializer_p' )
cv_qualifier_p.setName( 'cv_qualifier_p' )
cv_qualifier_seq_p.setName( 'cv_qualifier_seq_p' )
decl_specifiers_p.setName( 'decl_specifiers_p' )
declaration_pi.setName( 'declaration_pi' )
declaration_seq_pi.setName( 'declaration_seq_pi' )
declaration_statement_p.setName( 'declaration_statement_p' )
declarator_id_p.setName( 'declarator_id_p' )
declarator_p.setName( 'declarator_p' )
delete_expression_p.setName( 'delete_expression_p' )
direct_abstract_declarator_array_suffix_p.setName( 'direct_abstract_declarator_array_suffix_p' )
direct_abstract_declarator_fn_suffix_p.setName( 'direct_abstract_declarator_fn_suffix_p' )
direct_abstract_declarator_p.setName( 'direct_abstract_declarator_p' )
direct_declarator_p.setName( 'direct_declarator_p' )
direct_new_declarator_p.setName( 'direct_new_declarator_p' )
elaborated_type_specifier_p.setName( 'elaborated_type_specifier_p' )
end_p.setName( 'end_p' )
enum_name_p.setName( 'enum_name_p' )
enum_specifier_p.setName( 'enum_specifier_p' )
enumerator_definition_p.setName( 'enumerator_definition_p' )
enumerator_p.setName( 'enumerator_p' )
eos_p.setName( 'eos_p' )
equality_expression_p.setName( 'equality_expression_p' )
exception_declaration_p.setName( 'exception_declaration_p' )
exception_specification_pi.setName( 'exception_specification_pi' )
exclusive_or_expression_p.setName( 'exclusive_or_expression_p' )
explicit_instantiation_p.setName( 'explicit_instantiation_p' )
explicit_specialization_p.setName( 'explicit_specialization_p' )
expression_list_pi.setName( 'expression_list_pi' )
expression_pi.setName( 'expression_pi' )
expression_statement_p.setName( 'expression_statement_p' )
for_init_statement_p.setName( 'for_init_statement_p' )
forward_typename_id_p.setName( 'forward_typename_id_p' )
forwarded_type_id_p.setName( 'forwarded_type_id_p' )
friend_decl_specifiers_p.setName( 'friend_decl_specifiers_p' )
function_body_p.setName( 'function_body_p' )
function_declarator_p.setName( 'function_declarator_p' )
function_definition_pi.setName( 'function_definition_pi' )
function_definition_start_p.setName( 'function_definition_start_p' )
function_return_type_p.setName( 'function_return_type_p' )
function_specifier_p.setName( 'function_specifier_p' )
function_try_block_pi.setName( 'function_try_block_pi' )
handler_p.setName( 'handler_p' )
handler_seq_p.setName( 'handler_seq_p' )
id_expression_pi.setName( 'id_expression_pi' )
id_p.setName( 'id_p' )
inclusive_or_expression_p.setName( 'inclusive_or_expression_p' )
init_declarator_list_p.setName( 'init_declarator_list_p' )
init_declarator_p.setName( 'init_declarator_p' )
initializer_clause_p.setName( 'initializer_clause_p' )
initializer_list_p.setName( 'initializer_list_p' )
initializer_p.setName( 'initializer_p' )
instance_accessor_p.setName( 'instance_accessor_p' )
iteration_statement_p.setName( 'iteration_statement_p' )
jump_statement_p.setName( 'jump_statement_p' )
labeled_statement_p.setName( 'labeled_statement_p' )
linkage_specification_begin_p.setName( 'linkage_specification_begin_p' )
linkage_specification_p.setName( 'linkage_specification_p' )
literal_expression_p.setName( 'literal_expression_p' )
logical_and_expression_p.setName( 'logical_and_expression_p' )
logical_or_expression_p.setName( 'logical_or_expression_p' )
mem_initializer_p.setName( 'mem_initializer_p' )
member_access_suffix_p.setName( 'member_access_suffix_p' )
member_declaration_p.setName( 'member_declaration_p' )
member_declarator_list_p.setName( 'member_declarator_list_p' )
member_declarator_p.setName( 'member_declarator_p' )
member_specification_p.setName( 'member_specification_p' )
multiplicative_expression_p.setName( 'multiplicative_expression_p' )
named_namespace_definition_p.setName( 'named_namespace_definition_p' )
namespace_alias_definition_p.setName( 'namespace_alias_definition_p' )
namespace_body_p.setName( 'namespace_body_p' )
namespace_definition_p.setName( 'namespace_definition_p' )
namespace_name_pi.setName( 'namespace_name_pi' )
nested_name_specifier_pi.setName( 'nested_name_specifier_pi' )
new_declarator_p.setName( 'new_declarator_p' )
new_expression_p.setName( 'new_expression_p' )
new_initializer_p.setName( 'new_initializer_p' )
new_placement_p.setName( 'new_placement_p' )
new_type_id_alt_p.setName( 'new_type_id_alt_p' )
new_type_id_p.setName( 'new_type_id_p' )
new_type_pi.setName( 'new_type_pi' )
operator_function_id_pi.setName( 'operator_function_id_pi' )
operator_p.setName( 'operator_p' )
opt_parameter_decl_assign_p.setName( 'opt_parameter_decl_assign_p' )
parameter_declaration_clause_p.setName( 'parameter_declaration_clause_p' )
parameter_declaration_p.setName( 'parameter_declaration_p' )
parameter_type_p.setName( 'parameter_type_p' )
pm_expression_p.setName( 'pm_expression_p' )
post_incdec_suffix_p.setName( 'post_incdec_suffix_p' )
postfix_expression_p.setName( 'postfix_expression_p' )
primary_expression_p.setName( 'primary_expression_p' )
pseudo_destructor_name_p.setName( 'pseudo_destructor_name_p' )
ptr_operator_pi.setName( 'ptr_operator_pi' )
pure_specifier_p.setName( 'pure_specifier_p' )
qualified_id_p.setName( 'qualified_id_p' )
qualified_namespace_specifier_p.setName( 'qualified_namespace_specifier_p' )
relational_expression_p.setName( 'relational_expression_p' )
selection_statement_p.setName( 'selection_statement_p' )
shift_expression_p.setName( 'shift_expression_p' )
simple_declaration_p.setName( 'simple_declaration_p' )
simple_type_specifier_p.setName( 'simple_type_specifier_p' )
standard_decl_specifier_impl_p.setName( 'standard_decl_specifier_impl_p' )
standard_decl_specifiers_p.setName( 'standard_decl_specifiers_p' )
statement_p.setName( 'statement_p' )
storage_class_specifier_p.setName( 'storage_class_specifier_p' )
template_argument_list_p.setName( 'template_argument_list_p' )
template_argument_p.setName( 'template_argument_p' )
template_declaration_p.setName( 'template_declaration_p' )
template_id_pi.setName( 'template_id_pi' )
template_name_p.setName( 'template_name_p' )
template_name_specifier_p.setName( 'template_name_specifier_p' )
template_parameter_list_p.setName( 'template_parameter_list_p' )
template_parameter_p.setName( 'template_parameter_p' )
throw_expression_pi.setName( 'throw_expression_pi' )
translation_unit_pi.setName( 'translation_unit_pi' )
try_block_pi.setName( 'try_block_pi' )
type_id_expression_p.setName( 'type_id_expression_p' )
type_id_list_p.setName( 'type_id_list_p' )
type_id_pi.setName( 'type_id_pi' )
type_name_pi.setName( 'type_name_pi' )
type_parameter_p.setName( 'type_parameter_p' )
type_specifier_seq_pi.setName( 'type_specifier_seq_pi' )
typedef_decl_specifiers_p.setName( 'typedef_decl_specifiers_p' )
typedef_name_p.setName( 'typedef_name_p' )
unary_expression_p.setName( 'unary_expression_p' )
unary_operator_p.setName( 'unary_operator_p' )
unnamed_namespace_definition_p.setName( 'unnamed_namespace_definition_p' )
unqualified_id_pi.setName( 'unqualified_id_pi' )
user_type_specifier_p.setName( 'user_type_specifier_p' )
using_declaration_p.setName( 'using_declaration_p' )
using_directive_p.setName( 'using_directive_p' )
using_id_p.setName( 'using_id_p' )
workaround_type_specifier_pi.setName( 'workaround_type_specifier_pi' )
--- NEW FILE: grammartest.py ---
import unittest
import mock
import parser
import grammar
import parsingtracker
import string
import tokenizer
from parsertesthelper import *
# * need to modify binary_operator right node so that the operator is a child of binary_operator node.
# * end_p to terminate translation unit
# * error recovery using subparsing and special 'error' matching rules. Will probably need a special parser to
# skip 'a statement' (skip everything until next semi-colon while balancing braces and curly braces. stop if
# closing a curly-brace (means that the end of the block is found) )
def unqualified_id_n( name ):
return TestNode( 'unqualified_id', id_n(name) )
def fundamental_type_n( types ):
return TestNode( 'fundamental_type_specifier', *types.split() )
def fundamental_type_specifier_n( types ):
return TestNode( 'type_specifier', fundamental_type_n( types ) )
def declarator_id_n( *ids ):
return TestNode( 'declarator_id', TestNode( 'unqualified_id', *ids ) )
class GrammarTest(unittest.TestCase,ParserTestHelperMixIn):
"""GrammarTest tests"""
def setUp( self ):
pass
def tearDown( self ):
pass
# Expression
def testLiteralExpression( self ):
self.checkParser( '1', grammar.literal_expression_p )
self.checkParser( '1.0', grammar.literal_expression_p )
self.checkParser( '"abc"', grammar.literal_expression_p )
self.checkParser( "'a'", grammar.literal_expression_p )
self.checkParser( "true", grammar.literal_expression_p )
self.checkParser( "false", grammar.literal_expression_p )
self.checkProducedTree( 'true', grammar.literal_expression_p, ('literal','true') )
self.checkProducedTree( '123.456', grammar.literal_expression_p, ('literal','123.456') )
def testUnqualifiedId( self ):
self.checkProducedNodes( 'xyz', grammar.unqualified_id_pi, id_n( 'xyz' ) )
self.checkProducedNodes( 'operator =', grammar.unqualified_id_pi,
TestNode( 'operator_function_ref', 'operator', '=' ) )
self.checkProducedNodes( 'operator int', grammar.unqualified_id_pi,
TestNode( 'conversion_function_ref',
'operator',
TestNode( 'conversion_type', fundamental_type_specifier_n('int') ) ) )
self.checkProducedNodes( '~ClassName()', grammar.unqualified_id_pi,
TestNode( 'destructor_ref', '~', id_n('ClassName') ) )
def testQualifiedId( self ):
self.checkProducedNodes( '::NamespaceName::ClassName::xyz', grammar.qualified_id_p,
TestNode( 'qualified_id',
'::', id_n('NamespaceName'), '::', id_n('ClassName'), '::', id_n('xyz') ) )
self.checkProducedNodes( 'Name::template Class::xyz', grammar.qualified_id_p,
TestNode( 'qualified_id',
id_n('Name'), '::', 'template', id_n('Class'), '::', id_n('xyz') ) )
self.checkProducedNodes( '::xyz', grammar.qualified_id_p,
TestNode( 'qualified_id', '::', id_n('xyz') ) )
self.checkProducedNodes( '::operator =', grammar.qualified_id_p,
TestNode( 'qualified_id',
'::',
TestNode( 'operator_function_ref', 'operator', '=' ) ) )
self.checkProducedNodes( '::operator int', grammar.qualified_id_p,
TestNode( 'qualified_id',
'::',
TestNode( 'conversion_function_ref',
'operator',
TestNode( 'conversion_type', fundamental_type_specifier_n('int') ) ) ) )
def testPrimaryExpression( self ):
# require: literal_expression_p, qualified_id_p, unqualified_id_pi, id_expression_pi, operator_function_id_p, conversion_function_id_p
self.checkProducedTree( '1', grammar.primary_expression_p, ('literal','1') )
self.checkProducedTree( 'this', grammar.primary_expression_p, ('this_ref','this') )
self.checkProducedNodes( '(this)', grammar.primary_expression_p,
TestNode( 'braced_expr',
'(',
TestNode( 'expression', ('this_ref','this') ),
')' ) )
self.checkParser( '::my_id', grammar.primary_expression_p )
self.checkParser( 'MyNamespace::my_var', grammar.primary_expression_p )
self.checkParser( '::MyNamespace::MyClass::member_', grammar.primary_expression_p )
self.checkParser( '::operator =', grammar.primary_expression_p )
self.checkParser( 'Base::operator =', grammar.primary_expression_p )
self.checkProducedNodes( 'xyz', grammar.primary_expression_p,
TestNode( 'unqualified_id', id_n( 'xyz' ) ) )
self.checkProducedNodes( 'operator =', grammar.primary_expression_p,
TestNode( 'unqualified_id',
TestNode( 'operator_function_ref', 'operator', '=' ) ) )
self.checkProducedNodes( 'operator int', grammar.primary_expression_p,
TestNode( 'unqualified_id',
TestNode( 'conversion_function_ref',
'operator',
TestNode( 'conversion_type', fundamental_type_specifier_n('int') ) ) ) )
self.checkProducedNodes( '~ClassName()', grammar.primary_expression_p,
TestNode( 'unqualified_id',
TestNode( 'destructor_ref', '~', id_n('ClassName') ) ) )
def testPostFixExpressionSuffix( self ):
self.checkProducedNodes( '[123]', grammar.array_operator_suffix_p,
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') )
self.checkParser( '()', grammar.call_operator_suffix_p )
self.checkParser( '(123)', grammar.call_operator_suffix_p )
self.checkParser( '(123,456)', grammar.call_operator_suffix_p )
self.checkParser( '.my_member', grammar.member_access_suffix_p )
self.checkParser( '->my_member', grammar.member_access_suffix_p )
self.checkParser( '->template Parent::my_member', grammar.member_access_suffix_p )
self.checkParser( '.~MyClass', grammar.member_access_suffix_p )
self.checkParser( '.::Base::~MyClass', grammar.member_access_suffix_p )
self.checkParser( '.MyParent::~MyClass', grammar.member_access_suffix_p )
self.checkParser( '++', grammar.post_incdec_suffix_p )
self.checkParser( '--', grammar.post_incdec_suffix_p )
def testPostFixExpression( self ):
# require: primary_expression_p
self.checkParser( '(this)', grammar.postfix_expression_p )
self.checkParser( 'my_id', grammar.postfix_expression_p )
self.checkProducedNodes( 'arr[1][2]', grammar.postfix_expression_p,
TestNode( 'array_access',
TestNode( 'array_access',
JokerTestNode( 'unqualified_id' ),
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ), #[1]
TestNode('array_index', '[', JokerTestNode( 'expression' ), ']') ) ) #[2]
self.checkProducedTree( 'this', grammar.postfix_expression_p, ('this_ref','this') )
self.checkParser( 'int(1234)', grammar.postfix_expression_p )
self.checkParser( 'MyClass(1234)', grammar.postfix_expression_p )
self.checkParser( 'MyNamespace::MyClass(1234)', grammar.postfix_expression_p )
self.checkParser( 'typeid( 1234.0 )', grammar.postfix_expression_p )
self.checkParser( 'typeid( int )', grammar.postfix_expression_p )
self.checkParser( 'static_cast<int>(1234)', grammar.postfix_expression_p )
self.checkParser( 'dynamic_cast<Derived>(p)', grammar.postfix_expression_p )
self.checkParser( 'my_array[1]', grammar.postfix_expression_p )
self.checkParser( 'my_function(1)', grammar.postfix_expression_p )
self.checkParser( 'my_instance->my_member', grammar.postfix_expression_p )
self.checkParser( 'my_instance.BaseClass::my_member', grammar.postfix_expression_p )
self.checkParser( 'my_instance->~MyClass', grammar.postfix_expression_p )
self.checkParser( 'my_instance.MyClass::~MyClass()', grammar.postfix_expression_p, 0 )
self.checkParser( 'my_variable++', grammar.postfix_expression_p )
self.checkParser( 'my_variable--', grammar.postfix_expression_p )
self.checkParser( 'my_instance->my_member++', grammar.postfix_expression_p )
self.checkParser( '(my_instance->my_member)->my_member++', grammar.postfix_expression_p )
self.checkParser( 'typename ::Base::attribute(1,2)', grammar.postfix_expression_p )
def testDeleteExpressionParser( self ):
self.checkParser( 'delete p', grammar.delete_expression_p )
self.checkParser( '::delete p', grammar.delete_expression_p )
self.checkParser( 'delete ++p', grammar.delete_expression_p )
self.checkProducedNodes( '::delete [] p', grammar.delete_expression_p,
TestNode( 'delete_operator', '::', 'delete', '[', ']', unqualified_id_n('p') ) )
self.checkParser( 'delete x.z', grammar.delete_expression_p )
def testNewTypeIdParser( self ):
self.checkParser( 'int', grammar.new_type_id_alt_p )
self.checkParser( 'int *', grammar.new_type_id_alt_p )
self.checkParser( 'UserType[10][5]', grammar.new_type_id_alt_p )
self.checkParser( '(UserType*[10][5])', grammar.new_type_id_alt_p )
self.checkParserFail( 'const UserType', grammar.new_type_id_alt_p )
def testNewExpressionParser( self ):
self.checkProducedNodes( 'new int', grammar.new_expression_p,
TestNode( 'new_operator', 'new',
TestNode('new_type_id', fundamental_type_n('int') ) ) )
self.checkParser( 'new int(3)', grammar.new_expression_p )
self.checkProducedNodes( '::new (__FILE__,__LINE__) int(3)', grammar.new_expression_p,
TestNode( 'new_operator', '::', 'new',
TestNode( 'new_placement', '(',
TestNode( 'expression_list',
TestNode( 'assignment_expression',
unqualified_id_n('__FILE__') ),
',',
TestNode( 'assignment_expression',
unqualified_id_n('__LINE__') ) ),
')'),
TestNode( 'new_type_id', fundamental_type_n('int') ),
TestNode( 'new_initializer', '(',
TestNode( 'expression_list',
TestNode( 'assignment_expression',
('literal','3') ) ),
')' ) ) )
self.checkParser( 'new (int)(3)', grammar.new_expression_p )
self.checkParser( 'new int[10](7)', grammar.new_expression_p )
self.checkParser( 'new MyType[10](7)', grammar.new_expression_p )
self.checkParser( 'new (UserType)(7)', grammar.new_expression_p )
self.checkParser( 'new (UserType(3))(UserType)(7)', grammar.new_expression_p )
# need more test concerning new_type_id (should be restricted to plain type without cv)
#expression_list structure for new_initializer need to be defined
def testUnaryExpressionParser( self ):
#require postfix_expression_p
self.checkProducedTree( 'this', grammar.unary_expression_p, ('this_ref','this') )
self.checkParser( '(this)', grammar.unary_expression_p )
self.checkProducedNodes( '++my_variable', grammar.unary_expression_p,
TestNode( 'unary_operator',
'++',
unqualified_id_n('my_variable') ) )
self.checkProducedNodes( '!my_variable', grammar.unary_expression_p,
TestNode( 'unary_operator',
'!',
unqualified_id_n('my_variable') ) )
self.checkParser( '--my_variable', grammar.unary_expression_p )
self.checkParser( '-my_variable', grammar.unary_expression_p )
self.checkProducedNodes( 'sizeof !my_variable', grammar.unary_expression_p,
TestNode( 'sizeof_operator',
'sizeof',
TestNode( 'unary_operator', '!',
unqualified_id_n('my_variable') ) ) )
self.checkProducedNodes( 'sizeof(++my_variable)', grammar.unary_expression_p,
TestNode( 'sizeof_operator',
'sizeof',
TestNode( 'braced_expr',
'(',
TestNode( 'expression',
TestNode( 'unary_operator', '++',
unqualified_id_n('my_variable') ) ),
')' ) ) )
self.checkParser( 'new int[10]', grammar.unary_expression_p )
self.checkParser( 'delete p', grammar.unary_expression_p )
self.checkParser( '(my_instance->my_member)->my_member++', grammar.unary_expression_p )
def testPMExpression( self ):
self.checkProducedTree( '1234', grammar.pm_expression_p, ('literal','1234') )
def testLogicalOrExpressionParser( self ):
self.checkProducedTree( '1234', grammar.logical_or_expression_p, ('literal','1234') )
self.checkParser( 'my_instance->*my_member', grammar.logical_or_expression_p )
self.checkParser( 'sizeof ++my_variable', grammar.logical_or_expression_p )
self.checkParser( '34 * 35', grammar.logical_or_expression_p )
self.checkProducedNodes( '34 * 35', grammar.logical_or_expression_p,
TestNode( 'binary_operator',
...
[truncated message content] |
|
From: <bl...@us...> - 2003-08-24 21:00:40
|
Update of /cvsroot/cpptool/rfta/doc In directory sc8-pr-cvs1:/tmp/cvs-serv3499/doc Modified Files: directories.txt Log Message: * added C++ parser in python (without C++ template for the time being), as well as a load of tests. Index: directories.txt =================================================================== RCS file: /cvsroot/cpptool/rfta/doc/directories.txt,v retrieving revision 1.1.1.1 retrieving revision 1.2 diff -C2 -d -r1.1.1.1 -r1.2 *** directories.txt 23 Oct 2002 20:25:13 -0000 1.1.1.1 --- directories.txt 24 Aug 2003 21:00:36 -0000 1.2 *************** *** 9,14 **** lib/ Contains .lib generated by rfta src/ Rfta sources - rftaparser/ Parser and AST library astdump/ AST dumper executable --- 9,16 ---- lib/ Contains .lib generated by rfta src/ Rfta sources astdump/ AST dumper executable + pyrfta/ Python binding for the rfta library + rftaparser/ Parser and AST library + rfta/ Code analysis and refactoring library |
|
From: <bl...@us...> - 2003-08-24 21:00:40
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv3499/src/pyrfta/test/rfta
Modified Files:
__init__.py alltests.py
Removed Files:
parser.py
Log Message:
* added C++ parser in python (without C++ template for the time being), as well as a load of tests.
Index: __init__.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/__init__.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** __init__.py 30 May 2003 07:28:02 -0000 1.2
--- __init__.py 24 Aug 2003 21:00:36 -0000 1.3
***************
*** 1,20 ****
! from _pyrfta import *
!
! def _identifier_str( self ):
! return self.identifier
!
! def _declarator_str( self ):
! return self.type.typeText + ' ' + self.name.identifier + self.typeSuffix.typeText
!
! def _declarator_expr_str( self ):
! s = 'DeclarationExpr<' + self.primaryType.typeText + " "
! sep = ''
! for decl in self.getDeclarators():
! s += sep
! s += str(decl)
! sep = ','
! return s + '>'
!
! Declarator.__str__ = _declarator_str
! DeclaratorExpression.__repr__ = _declarator_expr_str
! Identifier.__str__ = _identifier_str
\ No newline at end of file
--- 1,22 ----
! ##Commented out because dll are not compiled against Python 2.3
! ##
! ##from _pyrfta import *
! ##
! ##def _identifier_str( self ):
! ## return self.identifier
! ##
! ##def _declarator_str( self ):
! ## return self.type.typeText + ' ' + self.name.identifier + self.typeSuffix.typeText
! ##
! ##def _declarator_expr_str( self ):
! ## s = 'DeclarationExpr<' + self.primaryType.typeText + " "
! ## sep = ''
! ## for decl in self.getDeclarators():
! ## s += sep
! ## s += str(decl)
! ## sep = ','
! ## return s + '>'
! ##
! ##Declarator.__str__ = _declarator_str
! ##DeclaratorExpression.__repr__ = _declarator_expr_str
! ##Identifier.__str__ = _identifier_str
Index: alltests.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/alltests.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** alltests.py 31 Jul 2003 08:21:48 -0000 1.2
--- alltests.py 24 Aug 2003 21:00:36 -0000 1.3
***************
*** 9,14 ****
import unittest
! import rfta.codeanalysistest
! import rfta.parsertest
def suite():
--- 9,14 ----
import unittest
! #import rfta.codeanalysistest
! import parser.alltests
def suite():
***************
*** 16,20 ****
alltests = unittest.TestSuite()
# alltests.addTest(unittest.findTestCases(rfta.codeanalysistest))
! alltests.addTest(unittest.findTestCases(rfta.parsertest))
# for module in map(__import__, modules_to_test):
# alltests.addTest(unittest.findTestCases(module))
--- 16,20 ----
alltests = unittest.TestSuite()
# alltests.addTest(unittest.findTestCases(rfta.codeanalysistest))
! alltests.addTest(parser.alltests.suite())
# for module in map(__import__, modules_to_test):
# alltests.addTest(unittest.findTestCases(module))
***************
*** 23,24 ****
--- 23,26 ----
#if __name__ == '__main__':
# unittest.main(defaultTest='suite')
+ if __name__ == '__main__':
+ unittest.TextTestRunner().run(suite())
--- parser.py DELETED ---
|
|
From: <bl...@us...> - 2003-08-24 21:00:40
|
Update of /cvsroot/cpptool/rfta/src/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv3499/src/rfta
Modified Files:
RenameTempRefactoring.cpp
Log Message:
* added C++ parser in python (without C++ template for the time being), as well as a load of tests.
Index: RenameTempRefactoring.cpp
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/rfta/RenameTempRefactoring.cpp,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** RenameTempRefactoring.cpp 27 Apr 2003 22:02:03 -0000 1.16
--- RenameTempRefactoring.cpp 24 Aug 2003 21:00:37 -0000 1.17
***************
*** 26,30 ****
{
-
RenameTempRefactoring::RenameTempRefactoring( TextDocument &document,
int temporaryLocation )
--- 26,29 ----
|
|
From: <bl...@us...> - 2003-08-24 20:57:42
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser In directory sc8-pr-cvs1:/tmp/cvs-serv3049/parser Log Message: Directory /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser added to the repository |
|
From: <bl...@us...> - 2003-08-03 13:58:25
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv21046/src/pyrfta/test/rfta
Modified Files:
parser.py parsertest.py
Log Message:
* started testing expression parsers & fixed a few bug
* made debug trace optional
* disabled tree node result print
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** parser.py 3 Aug 2003 09:24:06 -0000 1.8
--- parser.py 3 Aug 2003 13:58:22 -0000 1.9
***************
*** 299,303 ****
class TokenProvider:
! def __init__( self, tokens ):
self.tokens = tokens
self.index = 0
--- 299,303 ----
class TokenProvider:
! def __init__( self, tokens, debug = False ):
self.tokens = tokens
[...1181 lines suppressed...]
--- 1368,1386 ----
# A.10, C++PL3 p.812
! throw_expression_p.setParser( symbol_f('throw') >> maybe_f( assignment_expression_p ) )
! exception_declaration_p = (type_specifier_seq_p >> maybe_f( declarator_p | abstract_declarator_p )) \
| '...'
! handler_p = symbol_f('catch') >> '(' >> exception_declaration_p >> ')' >> compound_statement_p
! handler_seq_p = repeat_f( 1, handler_p )
! type_id_list_p = list_f( nested_id_p, ',' )
! exception_specification_p.setParser( symbol_f( 'throw' ) >> '(' >> maybe_f( type_id_list_p ) >> ')' )
! try_block_p.setParser( symbol_f('try') >> compound_statement_p >> handler_seq_p )
! function_try_block_p.setParser( symbol_f('try') >> maybe_f(ctor_initializer_p) >> function_body_p >> handler_seq_p )
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** parsertest.py 3 Aug 2003 08:50:30 -0000 1.6
--- parsertest.py 3 Aug 2003 13:58:22 -0000 1.7
***************
*** 121,149 ****
self.scanner = None
! def testParseBasicParsers( self ):
! self.checkParser( "MyClass", parser.id_p )
! self.checkParser( ",", parser.symbol_p(',') )
! self.checkParser( "MyClass MyItem", parser.repeat_p( 0, parser.id_p ) )
! self.checkParser( "v1,v2,v3", parser.list_p( parser.id_p, ',' ) )
! self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
! self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
! self.checkParser( "class MyClass : public MyParent, MyParent2 { };", parser.class_decl_p )
!
! proxy_test_p = parser.proxy_p()
! proxy_test_p.setParser( parser.id_p )
! self.checkParser( "otot", proxy_test_p )
!
! self.checkParser( "namespace fs = boost::filesystem;", parser.namespace_alias_definition_p )
! self.checkParser( "using typename boost::filesystem::path;", parser.using_declaration_p )
! self.checkParser( "using boost::filesystem::path;", parser.using_declaration_p )
! self.checkParser( "using namespace boost::filesystem;", parser.using_directive_p )
# self.checkParser( "namespace { } ", parser.namespace_definition_p )
## self.checkParser( "namespace MyWork { } ", parser.namespace_definition_p )
# self.checkParser( 'extern "C" { }', parser.declaration_p )
! def checkParser( self, text, tested_parser ):
tokens = self.scanner.tokenize( text )
! scanner = parser.TokenProvider( tokens )
match = tested_parser.parse( scanner )
self.assert_( match, "parser should have matched source" )
--- 121,178 ----
self.scanner = None
! ## def testParseBasicParsers( self ):
! ## self.checkParser( "MyClass", parser.id_p )
! ## self.checkParser( ",", parser.symbol_p(',') )
! ## self.checkParser( "MyClass MyItem", parser.repeat_p( 0, parser.id_p ) )
! ## self.checkParser( "v1,v2,v3", parser.list_p( parser.id_p, ',' ) )
! ## self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
! ## self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
! ## self.checkParser( "class MyClass : public MyParent, MyParent2 { };", parser.class_decl_p )
! ##
! ## proxy_test_p = parser.proxy_p()
! ## proxy_test_p.setParser( parser.id_p )
! ## self.checkParser( "otot", proxy_test_p )
! ##
! ## self.checkParser( "namespace fs = boost::filesystem;", parser.namespace_alias_definition_p )
! ## self.checkParser( "using typename boost::filesystem::path;", parser.using_declaration_p )
! ## self.checkParser( "using boost::filesystem::path;", parser.using_declaration_p )
! ## self.checkParser( "using namespace boost::filesystem;", parser.using_directive_p )
# self.checkParser( "namespace { } ", parser.namespace_definition_p )
## self.checkParser( "namespace MyWork { } ", parser.namespace_definition_p )
# self.checkParser( 'extern "C" { }', parser.declaration_p )
+
+ def testLiteralExpression( self ):
+ self.checkParser( '1', parser.literal_expression_p )
+ self.checkParser( '1.0', parser.literal_expression_p )
+ self.checkParser( '"abc"', parser.literal_expression_p )
+ self.checkParser( "'a'", parser.literal_expression_p )
+
+ def testPrimaryExpression( self ):
+ # require: literal_expression_p, qualified_id_p, unqualified_id_p, id_expression_p, operator_function_id_p, conversion_function_id_p
+ self.checkParser( '1', parser.primary_expression_p )
+ self.checkParser( 'this', parser.primary_expression_p )
+ self.checkParser( '::my_id', parser.primary_expression_p )
+ self.checkParser( '::MyNamespace::MyClass::member_', parser.primary_expression_p )
+ self.checkParser( '::operator =', parser.primary_expression_p )
+ self.checkParser( '(this)', parser.primary_expression_p )
+ self.checkParser( 'my_var', parser.primary_expression_p )
+ self.checkParser( 'MyNamespace::my_var', parser.primary_expression_p )
+ self.checkParser( 'operator =', parser.primary_expression_p )
+ self.checkParser( 'operator int', parser.primary_expression_p )
+
+ def testPostFixExpression( self ):
+ # require: primary_expression_p
+ self.checkParser( '(this)', parser.postfix_expression_p )
+ self.checkParser( 'my_array', parser.postfix_expression_p )
+ # => fail, need to introduce a new 'longuest alternative' parser
+ # self.checkParser( 'my_array[1]', parser.postfix_expression_p, True ) # also need more complex expression in index
! def testExpressionParsers( self ):
! pass
!
! def checkParser( self, text, tested_parser, debug = False ):
tokens = self.scanner.tokenize( text )
! scanner = parser.TokenProvider( tokens, debug )
match = tested_parser.parse( scanner )
self.assert_( match, "parser should have matched source" )
***************
*** 152,158 ****
remaining_tokens.append( scanner.getNextToken() )
self.assertEqual( [], remaining_tokens )
! if match.getValue():
! print "Parsing '%s'" % text
! match.getValue().prettyPrint( 4 )
return match.getValue()
--- 181,187 ----
remaining_tokens.append( scanner.getNextToken() )
self.assertEqual( [], remaining_tokens )
! ## if match.getValue():
! ## print "Parsing '%s'" % text
! ## match.getValue().prettyPrint( 4 )
return match.getValue()
|
|
From: <bl...@us...> - 2003-08-03 09:24:09
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv13447/src/pyrfta/test/rfta
Modified Files:
parser.py
Log Message:
* added grammar for statements
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** parser.py 3 Aug 2003 08:50:30 -0000 1.7
--- parser.py 3 Aug 2003 09:24:06 -0000 1.8
***************
*** 1065,1071 ****
# ############################## A.6: Statements ############################
! #dummy
! compound_statement_p = symbol_p('{') >> '}'
# ############################## A.7.1: Declarators ############################
--- 1065,1115 ----
# ############################## A.6: Statements ############################
+ # A.6, C++PL3 p.802-803
! block_declaration_p = proxy_p()
! declarator_p = proxy_p()
! simple_declaration_p = proxy_p()
! statement_p = proxy_p()
! try_block_p = proxy_p()
!
!
! condition_p = expression_p \
! | ( type_specifier_seq_p >> declarator_p >> '=' >> assignment_expression_p )
!
! expression_statement_p = maybe_p(expression_p) >> eos_p
!
! jump_statement_p = (one_of_p( 'break continue' ) >> eos_p) \
! | (symbol_p('return') >> maybe_p(expression_p) >> eos_p) \
! | (symbol_p('goto') >> id_p)
!
! for_init_statement_p = expression_statement_p \
! | simple_declaration_p
!
! iteration_statement_p = (symbol_p('while') >> '(' >> condition_p >> ')' >> statement_p) \
! | (symbol_p('do') >> statement_p >> 'while' >> '(' >> expression_p >> ')' >> eos_p) \
! | (symbol_p('for') >> '(' >> for_init_statement_p >> maybe_p( condition_p ) \
! >> eos_p >> maybe_p( expression_p ) >> ')' >> statement_p )
!
! selection_statement_p = ( symbol_p('if') >> '(' >> condition_p >> ')' >> statement_p \
! >> maybe_p( symbol_p('else') >> statement_p ) ) \
! | ( symbol_p('switch') >> '(' >> condition_p >> ')' >> statement_p )
!
! compound_statement_p = symbol_p('{') >> repeat_p( 0, statement_p ) >> '}'
!
! labeled_statement_p = (id_p >> ':' >> statement_p) \
! | ( symbol_p('case') >> constant_expression_p >> ':' >> statement_p ) \
! | ( symbol_p('default') >> ':' >> statement_p )
!
! declaration_statement_p = block_declaration_p
!
! statement_p.setParser( labeled_statement_p
! | expression_statement_p
! | compound_statement_p
! | selection_statement_p
! | iteration_statement_p
! | jump_statement_p
! | declaration_statement_p
! | try_block_p )
!
# ############################## A.7.1: Declarators ############################
***************
*** 1075,1079 ****
class_specifier_p = proxy_p()
ctor_initializer_p = proxy_p()
- declarator_p = proxy_p()
decl_specifier_p = proxy_p() # forward declaration for recursion
exception_specification_p = proxy_p()
--- 1119,1122 ----
***************
*** 1223,1234 ****
| 'typedef' )
! simple_declaration_p = repeat_p( 0, decl_specifier_p ) >> maybe_p( init_declarator_list_p ) >> eos_p
# A.7, C++PL3 p.804
! block_declaration_p = asm_definition_p \
! | namespace_alias_definition_p \
! | using_declaration_p \
! | using_directive_p \
! | simple_declaration_p
declaration_p.setParser( block_declaration_p \
--- 1266,1277 ----
| 'typedef' )
! simple_declaration_p.setParser( repeat_p( 0, decl_specifier_p ) >> maybe_p( init_declarator_list_p ) >> eos_p )
# A.7, C++PL3 p.804
! block_declaration_p.setParser( asm_definition_p
! | namespace_alias_definition_p
! | using_declaration_p
! | using_directive_p
! | simple_declaration_p )
declaration_p.setParser( block_declaration_p \
***************
*** 1322,1326 ****
exception_specification_p.setParser( symbol_p( 'throw' ) >> '(' >> maybe_p( type_id_list_p ) >> ')' )
! try_block_p = symbol_p('try') >> compound_statement_p >> handler_seq_p
function_try_block_p.setParser( symbol_p('try') >> maybe_p(ctor_initializer_p) >> function_body_p >> handler_seq_p )
--- 1365,1369 ----
exception_specification_p.setParser( symbol_p( 'throw' ) >> '(' >> maybe_p( type_id_list_p ) >> ')' )
! try_block_p.setParser( symbol_p('try') >> compound_statement_p >> handler_seq_p )
function_try_block_p.setParser( symbol_p('try') >> maybe_p(ctor_initializer_p) >> function_body_p >> handler_seq_p )
|
|
From: <bl...@us...> - 2003-08-03 08:50:33
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv9037/src/pyrfta/test/rfta
Modified Files:
generateparserdebuginfo.py parser.py parsertest.py
Log Message:
* added grammar for expression
* added one_of_p() parser
* added some missing symbols to tokenizer
Index: generateparserdebuginfo.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/generateparserdebuginfo.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** generateparserdebuginfo.py 2 Aug 2003 21:29:08 -0000 1.1
--- generateparserdebuginfo.py 3 Aug 2003 08:50:30 -0000 1.2
***************
*** 3,7 ****
import string
! factories = string.split( 'make_p terminal_p symbol_p maybe_p repeat_p list_p anyuntil_p proxy_p' )
for var in dir(parser):
--- 3,7 ----
import string
! factories = string.split( 'make_p terminal_p symbol_p maybe_p repeat_p list_p anyuntil_p proxy_p one_of_p' )
for var in dir(parser):
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** parser.py 2 Aug 2003 21:29:08 -0000 1.6
--- parser.py 3 Aug 2003 08:50:30 -0000 1.7
***************
*** 83,87 ****
( FLOAT, '([0-9]+)?\\.[0-9]+(e[-+]?[0-9]+)?[fFL]?' ),
( FLOAT, '[0-9]+e[-+]?[0-9]+[fFL]?' ),
! ( SYMBOL, self._makeChoice( ':: << >> <= >= \\|\\| && &= \\|= \\+= -= /= \\*= \\^=' ) ),
( SYMBOL, self._makeChoice( '\\( \\) \\[ \\] \\{ \\} ~ / - \\+ / \\* = \\^ % < > & \\| ; , : \\?' ) ),
( CPP_COMMENT, '//' ),
--- 83,87 ----
( FLOAT, '([0-9]+)?\\.[0-9]+(e[-+]?[0-9]+)?[fFL]?' ),
( FLOAT, '[0-9]+e[-+]?[0-9]+[fFL]?' ),
! ( SYMBOL, self._makeChoice( '>>= <<= :: << >> <= >= \\|\\| && &= \\|= \\+= -= /= \\*= \\^= %=' ) ),
( SYMBOL, self._makeChoice( '\\( \\) \\[ \\] \\{ \\} ~ / - \\+ / \\* = \\^ % < > & \\| ; , : \\?' ) ),
( CPP_COMMENT, '//' ),
***************
*** 698,701 ****
--- 698,722 ----
self.parser = parser
+ class OneOfParser(Parser):
+ def __init__( self, symbols ):
+ Parser.__init__( self )
+ self.symbols = string.split( symbols )
+
+ def parse( self, scanner ):
+ scanner.enterParser( self )
+ token = scanner.getNextToken()
+ if token.getType() != SYMBOL or token.getText() not in self.symbols:
+ return self.notMatched( scanner )
+ return self.matched( scanner, TokenNode( token ) )
+
+ def __str__( self ):
+ return 'one_of(%s)' % string.join( self.symbols, ' ' )
+
+ def useShortDebugTrace( self ):
+ return True
+
+ def getDebugInfo( self ):
+ return str(self)
+
class InlineChildAction(Action):
def __init__( self, node_index ):
***************
*** 802,805 ****
--- 823,829 ----
return ProxyParser()
+ def one_of_p( symbols ):
+ return OneOfParser( symbols )
+
id_p = TerminalParser( IDENTIFIER )
***************
*** 889,905 ****
# ############################## A.5: Expressions ############################
# A.5, C++PL3 p.798-802
- constant_expression_p = terminal_p(INTEGER) | terminal_p(CHAR) # dummy
! # dummy
! assignment_expression_p = constant_expression_p
! ##assignment_expression_p = proxy_p()
##
! ##assignment_expression_p.setParser( conditional_expression_p \
! ## | ( logical_or_expression_p >> assignement_operator_p >> assignment_expression_p ) \
! ## | throw_expression_p )
! # dummy
! expression_list_p = constant_expression_p
# ############################## A.6: Statements ############################
--- 913,1066 ----
# ############################## A.5: Expressions ############################
# A.5, C++PL3 p.798-802
! operator_function_id_p = proxy_p()
! conversion_function_id_p = proxy_p()
! simple_type_specifier_p = proxy_p()
! ptr_operator_p = proxy_p()
! type_specifier_seq_p = proxy_p()
! throw_expression_p = proxy_p()
! expression_p = proxy_p()
! expression_list_p = proxy_p()
! cast_expression_p = proxy_p()
! assignment_expression_p = proxy_p()
! unary_expression_p = proxy_p()
! constant_expression_p = proxy_p()
!
! literal_expression_p = terminal_p(INTEGER) | terminal_p(CHAR) | terminal_p(FLOAT)
!
! # Notes: qualified_id can not include 'template' => A::template B::zozo is not parsed (IT SHOULD BE)
! qualified_id_p = nested_id_p >> maybe_p( symbol_p('template' ) >> id_p )
!
! # Notes: destructor and template id not included
! unqualified_id_p = id_p \
! | operator_function_id_p \
! | conversion_function_id_p
! # | ( symbol_p('~') >> id_p ) \
! # | template_id_p
!
!
! id_expression_p = unqualified_id_p | qualified_id_p
!
! primary_expression_p = literal_expression_p \
! | 'this' \
! | (symbol_p('::') >> id_p) \
! | (symbol_p('::') >> operator_function_id_p) \
! | (symbol_p('::') >> qualified_id_p) \
! | (symbol_p('(') >> expression_p >> ')' ) \
! | id_expression_p
!
! # Notes: missing template in scope case
! pseudo_destructor_name_p = maybe_p( nested_id_p >> '::' ) >> symbol_p('~') >> id_p
!
! instance_accessor_p = one_of_p('. ->')
!
! cast_keywords_p = one_of_p('dynamic_cast static_cast reinterpret_cast const_cast')
!
! # Notes: could not find actual grammar declaration. Fill in with a dummy rule
! type_id_p = nested_id_p
!
! # Notes: not included:
! # typename with template in scope
! #
! postfix_expression_p = proxy_p()
! postfix_expression_p.setParser( primary_expression_p
! | (postfix_expression_p >> '[' >> expression_p >> ']') # operator []
! | (postfix_expression_p >> '(' >> maybe_p( expression_list_p ) >> ')') # operator ()
! | (simple_type_specifier_p >> '(' >> maybe_p( expression_list_p ) >> ')') # simple type construction
! | (symbol_p('typename') >> nested_id_p >> '(' >> maybe_p( expression_list_p ) >> ')') # template construction
! | (postfix_expression_p >> instance_accessor_p >> maybe_p( 'template' ) >> maybe_p( '::' ) >> id_expression_p) # instance . id or instance -> id
! | (postfix_expression_p >> instance_accessor_p >> pseudo_destructor_name_p ) # destructor call
! | (postfix_expression_p >> (symbol_p('++') | '--'))
! | (cast_keywords_p >> '<' >> type_id_p >> '>' >> '(' >> expression_p >> ')')
! | (symbol_p('typeid') >> '(' >> expression_p >> ')')
! | (symbol_p('typeid') >> '(' >> type_id_p >> ')') )
!
! cast_expression_p = unary_expression_p \
! | (symbol_p('(') >> type_id_p >> ')')
!
! delete_expression_p = maybe_p( '::') >> 'delete' >> maybe_p('[]') >> cast_expression_p
!
! new_initializer_p = symbol_p('(') >> maybe_p( expression_list_p ) >> ')'
!
! expression_list_p.setParser( list_p( assignment_expression_p, ',' ) )
!
! direct_new_declarator_p = repeat_p( 1, symbol_p('[') >> expression_p >> ']' ) \
! >> maybe_p( symbol_p('[') >> constant_expression_p >> ']' )
!
! new_declarator_p = proxy_p()
! new_declarator_p.setParser( (ptr_operator_p >> maybe_p( new_declarator_p ))
! | direct_new_declarator_p )
!
! new_type_id_p = type_specifier_seq_p >> maybe_p( new_declarator_p )
!
! new_placement_p = symbol_p( '(' ) >> expression_list_p >> ')'
!
! new_expression_p = (maybe_p( '::' ) >> 'new' >> maybe_p(new_placement_p) >> new_type_id_p >> maybe_p(new_initializer_p)) \
! | (maybe_p( '::' ) >> 'new' >> maybe_p(new_placement_p) >> '(' >> new_type_id_p >> ')' >> maybe_p(new_initializer_p))
!
! unary_operator_p = one_of_p( '* & + - ! ~' )
!
! unary_expression_p.setParser( postfix_expression_p
! | ( (symbol_p('++') | '--') >> cast_expression_p )
! | (unary_operator_p >> cast_expression_p)
! | (symbol_p( 'sizeof' ) >> unary_expression_p)
! | (symbol_p( 'sizeof' ) >> '(' >> unary_expression_p >> ')' )
! | new_expression_p
! | delete_expression_p )
!
! pm_expression_p = proxy_p()
! pm_expression_p.setParser( cast_expression_p
! | (pm_expression_p >> instance_accessor_p >> '*' >> cast_expression_p) )
!
! ##multiplicative_expression_p = proxy_p()
! ##multiplicative_expression_p.setParser( pm_expression_p
! ## | ( multiplicative_expression_p >> one_of_p( '* / %' ) >> pm_expression_p) )
##
! ##additive_expression_p = proxy_p()
! ##additive_expression_p.setParser( multiplicative_expression_p
! ## | ( additive_expression_p >> one_of_p( '+ -' ) >> multiplicative_expression_p ) )
! ##
! ##shift_expression_p = proxy_p()
! ##shift_expression_p.setParser( additive_expression_p
! ## | ( shift_expression_p >> one_of_p( '>> <<' ) >> additive_expression_p ) )
! ##
! ##relational_expression_p = proxy_p()
! ##relational_expression_p.setParser( shift_expression_p
! ## | ( relational_expression_p >> one_of_p( '< > <= >=' ) >> shift_expression_p ) )
! ##
! ##equality_expression_p = proxy_p()
! ##equality_expression_p.setParser( relational_expression_p
! ## | ( equality_expression_p >> one_of_p( '== !=' ) >> relational_expression_p ) )
! def binary_expr_p_factory( simple_expr_parser, symbols ):
! p = proxy_p()
! return simple_expr_parser | ( p >> one_of_p( symbols ) >> simple_expr_parser )
!
! multiplicative_expression_p = binary_expr_p_factory( pm_expression_p, '* / %' )
! additive_expression_p = binary_expr_p_factory( multiplicative_expression_p, '+ -' )
! shift_expression_p = binary_expr_p_factory( additive_expression_p, '<< >>' )
! relational_expression_p = binary_expr_p_factory( shift_expression_p, '< > <= >=' )
! equality_expression_p = binary_expr_p_factory( relational_expression_p, '== !=' )
! and_expression_p = binary_expr_p_factory( equality_expression_p, '&' )
! exclusive_or_expression_p = binary_expr_p_factory( and_expression_p, '^' )
! inclusive_or_expression_p = binary_expr_p_factory( exclusive_or_expression_p, '|' )
! logical_and_expression_p = binary_expr_p_factory( inclusive_or_expression_p, '&&' )
! logical_or_expression_p = binary_expr_p_factory( logical_and_expression_p, '||' )
!
! conditional_expression_p = proxy_p()
! conditional_expression_p.setParser( logical_or_expression_p
! | ( conditional_expression_p >> '?' >> expression_p >> ':' >> assignment_expression_p ) )
!
! assignment_operator_p = one_of_p( '= *= /= %= += -= >>= <<= &= ^= |=' )
!
! assignment_expression_p.setParser( conditional_expression_p
! | (logical_or_expression_p >> assignment_operator_p >> assignment_expression_p)
! | throw_expression_p )
!
! expression_p.setParser( assignment_expression_p
! | (expression_p >> ',' >> assignment_expression_p ) )
!
! constant_expression_p.setParser( conditional_expression_p )
# ############################## A.6: Statements ############################
***************
*** 918,922 ****
exception_specification_p = proxy_p()
function_try_block_p = proxy_p()
- ptr_operator_p = proxy_p()
abstract_declarator_p = proxy_p()
direct_abstract_declarator_p = proxy_p()
--- 1079,1082 ----
***************
*** 985,989 ****
init_declarator_list_p = list_p( init_declarator_p, ',' )
! type_specifier_seq_p = repeat_p( 1, type_specifier_p )
# ############################## A.7: Declaration ############################
--- 1145,1149 ----
init_declarator_list_p = list_p( init_declarator_p, ',' )
! type_specifier_seq_p.setParser( repeat_p( 1, type_specifier_p ) )
# ############################## A.7: Declaration ############################
***************
*** 1047,1052 ****
# - opt(::) opt(nested-name-specifier) type-name
# - opt(::) opt(nested-name-specifier) opt(tempalte) template-id
! simple_type_specifier_p = symbol_p('char') | 'wchar_t' | 'bool' | 'short' | 'int' | 'long' \
! | 'signed' | 'unsigned' | 'float' | 'double' | 'void'
storage_class_specifier_p = symbol_p('auto') | 'register' | 'static' | 'extern' | 'mutable'
--- 1207,1211 ----
# - opt(::) opt(nested-name-specifier) type-name
# - opt(::) opt(nested-name-specifier) opt(tempalte) template-id
! simple_type_specifier_p.setParser( one_of_p( 'char wchar_t bool short int long signed unsigned float double void' ) )
storage_class_specifier_p = symbol_p('auto') | 'register' | 'static' | 'extern' | 'mutable'
***************
*** 1127,1130 ****
--- 1286,1295 ----
# A.8.2, C++PL3 p.810
+ conversion_declarator_p = repeat_p( 1, ptr_operator_p )
+
+ conversion_type_id_p = type_specifier_seq_p >> maybe_p( conversion_declarator_p )
+
+ conversion_function_id_p.setParser( symbol_p('operator') >> conversion_type_id_p )
+
mem_initializer_id = nested_id_p
***************
*** 1133,1136 ****
--- 1298,1308 ----
ctor_initializer_p.setParser( symbol_p( ':' ) >> list_p( mem_initializer_p, ',' ) )
+ # ############### A.8.2: Classes, OverLoading ##################
+ # A.8.3, C++PL3 p.810
+
+ operator_p = one_of_p( '+ - * / % ^ & | ~ ! = < > += -= *= /= %= ^= &= |= << >> >>= <<= == != <= >= && || ++ -- , ->* -> () []') \
+ | ( one_of_p( 'new delete' ) >> maybe_p( '[]' ) )
+
+ operator_function_id_p.setParser( symbol_p('operator') >> operator_p )
***************
*** 1138,1142 ****
# A.10, C++PL3 p.812
! throw_expression_p = symbol_p('throw') >> maybe_p( assignment_expression_p )
exception_declaration_p = (type_specifier_seq_p >> maybe_p( declarator_p | abstract_declarator_p )) \
--- 1310,1314 ----
# A.10, C++PL3 p.812
! throw_expression_p.setParser( symbol_p('throw') >> maybe_p( assignment_expression_p ) )
exception_declaration_p = (type_specifier_seq_p >> maybe_p( declarator_p | abstract_declarator_p )) \
***************
*** 1161,1168 ****
--- 1333,1343 ----
access_level_spec_p.setName( 'access_level_spec_p' )
access_specifier_p.setName( 'access_specifier_p' )
+ additive_expression_p.setName( 'additive_expression_p' )
+ and_expression_p.setName( 'and_expression_p' )
array_type_p.setName( 'array_type_p' )
asm_definition_p.setName( 'asm_definition_p' )
asm_keyword_p.setName( 'asm_keyword_p' )
assignment_expression_p.setName( 'assignment_expression_p' )
+ assignment_operator_p.setName( 'assignment_operator_p' )
attribut_decl_p.setName( 'attribut_decl_p' )
attribut_modifiers_p.setName( 'attribut_modifiers_p' )
***************
*** 1172,1175 ****
--- 1347,1352 ----
block_declaration_p.setName( 'block_declaration_p' )
block_statement_p.setName( 'block_statement_p' )
+ cast_expression_p.setName( 'cast_expression_p' )
+ cast_keywords_p.setName( 'cast_keywords_p' )
class_body_statements_p.setName( 'class_body_statements_p' )
class_decl_p.setName( 'class_decl_p' )
***************
*** 1183,1188 ****
--- 1360,1369 ----
class_specifier_p.setName( 'class_specifier_p' )
compound_statement_p.setName( 'compound_statement_p' )
+ conditional_expression_p.setName( 'conditional_expression_p' )
constant_expression_p.setName( 'constant_expression_p' )
constant_initializer_p.setName( 'constant_initializer_p' )
+ conversion_declarator_p.setName( 'conversion_declarator_p' )
+ conversion_function_id_p.setName( 'conversion_function_id_p' )
+ conversion_type_id_p.setName( 'conversion_type_id_p' )
ctor_initializer_p.setName( 'ctor_initializer_p' )
cv_modifier_p.setName( 'cv_modifier_p' )
***************
*** 1194,1199 ****
--- 1375,1382 ----
declaration_seq_p.setName( 'declaration_seq_p' )
declarator_p.setName( 'declarator_p' )
+ delete_expression_p.setName( 'delete_expression_p' )
direct_abstract_declarator_p.setName( 'direct_abstract_declarator_p' )
direct_declarator_p.setName( 'direct_declarator_p' )
+ direct_new_declarator_p.setName( 'direct_new_declarator_p' )
elaborated_type_specifier_p.setName( 'elaborated_type_specifier_p' )
end_p.setName( 'end_p' )
***************
*** 1202,1208 ****
--- 1385,1394 ----
enumerator_p.setName( 'enumerator_p' )
eos_p.setName( 'eos_p' )
+ equality_expression_p.setName( 'equality_expression_p' )
exception_declaration_p.setName( 'exception_declaration_p' )
exception_specification_p.setName( 'exception_specification_p' )
+ exclusive_or_expression_p.setName( 'exclusive_or_expression_p' )
expression_list_p.setName( 'expression_list_p' )
+ expression_p.setName( 'expression_p' )
friend_decl_p.setName( 'friend_decl_p' )
function_body_p.setName( 'function_body_p' )
***************
*** 1214,1218 ****
--- 1400,1406 ----
handler_p.setName( 'handler_p' )
handler_seq_p.setName( 'handler_seq_p' )
+ id_expression_p.setName( 'id_expression_p' )
id_p.setName( 'id_p' )
+ inclusive_or_expression_p.setName( 'inclusive_or_expression_p' )
init_declarator_list_p.setName( 'init_declarator_list_p' )
init_declarator_p.setName( 'init_declarator_p' )
***************
*** 1220,1225 ****
--- 1408,1417 ----
initializer_list_p.setName( 'initializer_list_p' )
initializer_p.setName( 'initializer_p' )
+ instance_accessor_p.setName( 'instance_accessor_p' )
linkage_specification_begin_p.setName( 'linkage_specification_begin_p' )
linkage_specification_p.setName( 'linkage_specification_p' )
+ literal_expression_p.setName( 'literal_expression_p' )
+ logical_and_expression_p.setName( 'logical_and_expression_p' )
+ logical_or_expression_p.setName( 'logical_or_expression_p' )
mem_initializer_p.setName( 'mem_initializer_p' )
member_declaration_p.setName( 'member_declaration_p' )
***************
*** 1229,1232 ****
--- 1421,1425 ----
member_initializers_p.setName( 'member_initializers_p' )
member_specification_p.setName( 'member_specification_p' )
+ multiplicative_expression_p.setName( 'multiplicative_expression_p' )
named_namespace_definition_p.setName( 'named_namespace_definition_p' )
namespace_alias_definition_p.setName( 'namespace_alias_definition_p' )
***************
*** 1234,1245 ****
--- 1427,1452 ----
namespace_definition_p.setName( 'namespace_definition_p' )
nested_id_p.setName( 'nested_id_p' )
+ new_declarator_p.setName( 'new_declarator_p' )
+ new_expression_p.setName( 'new_expression_p' )
+ new_initializer_p.setName( 'new_initializer_p' )
+ new_placement_p.setName( 'new_placement_p' )
+ new_type_id_p.setName( 'new_type_id_p' )
null_statement_p.setName( 'null_statement_p' )
+ operator_function_id_p.setName( 'operator_function_id_p' )
+ operator_p.setName( 'operator_p' )
opt_parameter_decl_assign_p.setName( 'opt_parameter_decl_assign_p' )
optional_initializer_p.setName( 'optional_initializer_p' )
parameter_declaration_clause_p.setName( 'parameter_declaration_clause_p' )
parameter_declaration_p.setName( 'parameter_declaration_p' )
+ pm_expression_p.setName( 'pm_expression_p' )
+ postfix_expression_p.setName( 'postfix_expression_p' )
+ primary_expression_p.setName( 'primary_expression_p' )
+ pseudo_destructor_name_p.setName( 'pseudo_destructor_name_p' )
ptr_operator_p.setName( 'ptr_operator_p' )
pure_specifier_p.setName( 'pure_specifier_p' )
+ qualified_id_p.setName( 'qualified_id_p' )
qualified_namespace_specifier_p.setName( 'qualified_namespace_specifier_p' )
+ relational_expression_p.setName( 'relational_expression_p' )
+ shift_expression_p.setName( 'shift_expression_p' )
simple_declaration_p.setName( 'simple_declaration_p' )
simple_type_specifier_p.setName( 'simple_type_specifier_p' )
***************
*** 1250,1258 ****
--- 1457,1469 ----
try_block_p.setName( 'try_block_p' )
type_id_list_p.setName( 'type_id_list_p' )
+ type_id_p.setName( 'type_id_p' )
type_p.setName( 'type_p' )
type_specifier_p.setName( 'type_specifier_p' )
type_specifier_seq_p.setName( 'type_specifier_seq_p' )
type_suffix_p.setName( 'type_suffix_p' )
+ unary_expression_p.setName( 'unary_expression_p' )
+ unary_operator_p.setName( 'unary_operator_p' )
unnamed_namespace_definition_p.setName( 'unnamed_namespace_definition_p' )
+ unqualified_id_p.setName( 'unqualified_id_p' )
using_declaration_p.setName( 'using_declaration_p' )
using_directive_p.setName( 'using_directive_p' )
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** parsertest.py 2 Aug 2003 21:29:08 -0000 1.5
--- parsertest.py 3 Aug 2003 08:50:30 -0000 1.6
***************
*** 63,67 ****
def testSymbol( self ):
! symbols = string.split( ':: << >> <= >= || && &= |= += -= /= *= ^= ' \
'( ) [ ] { } ~ / - + / * = ^ % < > & | ; , : ? ' \
'if else break continue for while do void int' )
--- 63,67 ----
def testSymbol( self ):
! symbols = string.split( ':: << >> <= >= || && &= |= += -= /= *= ^= %= >>= <<= ' \
'( ) [ ] { } ~ / - + / * = ^ % < > & | ; , : ? ' \
'if else break continue for while do void int' )
|
|
From: <bl...@us...> - 2003-08-02 21:29:11
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv21750/src/pyrfta/test/rfta
Modified Files:
parser.py parsertest.py
Added Files:
generateparserdebuginfo.py
Log Message:
* added parser traces for debug
* script to generate parser name for debug (from variable)
--- NEW FILE: generateparserdebuginfo.py ---
# generate parser name for debug
import parser
import string
factories = string.split( 'make_p terminal_p symbol_p maybe_p repeat_p list_p anyuntil_p proxy_p' )
for var in dir(parser):
if var not in factories and len(var) > 2 and var[-2:] == '_p':
print "%s.setName( '%s' )" % (var,var)
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -C2 -d -r1.5 -r1.6
*** parser.py 2 Aug 2003 18:33:53 -0000 1.5
--- parser.py 2 Aug 2003 21:29:08 -0000 1.6
***************
*** 304,307 ****
--- 304,308 ----
self.states = []
self.end_stream_token = Token( END_STREAM, 0, 2+3+2, "<<end>>" )
+ self.parser_depth = 0
def hasMoreToken( self ):
***************
*** 325,328 ****
--- 326,386 ----
self.states.pop()
+ def enterParser( self, parser ):
+ if parser.isActionParser():
+ return
+
+ if not parser.useShortDebugTrace():
+ self.printDebugMessage( "Entering %s <= %s" % (self.getParserInfo(parser),self.miniScannerStateReport()), True )
+ self.parser_depth += 1
+
+ def exitParser( self, parser, message ):
+ if parser.isActionParser():
+ return
+
+ if parser.useShortDebugTrace():
+ self.printDebugMessage( "%s: %s" % (message,self.getParserShortInfo(parser)) )
+ return
+ heading = "Parsed"
+ else:
+ heading = "Exiting "
+ self.printDebugMessage( "%s: %s %s => %s" % (message,heading,self.getParserInfo(parser), self.miniScannerStateReport()) )
+ if not parser.useShortDebugTrace():
+ self.parser_depth -= 1
+
+ def getParserShortInfo( self, parser ):
+ info = parser.getName()
+ if not info:
+ info = parser.getDebugInfo()
+ return info
+
+ def getParserInfo( self, parser ):
+ parser_type = string.split( parser.__class__.__name__, '.' )[-1]
+ return "%s[%s]" % (self.getParserShortInfo(parser),parser_type)
+
+ def miniScannerStateReport( self ):
+ self.pushState()
+ next_tokens = []
+ for index in range(10):
+ if not self.hasMoreToken():
+ break
+ token = self.getNextToken()
+ if token.getType() == SYMBOL:
+ next_tokens.append( token.getText() )
+ else:
+ next_tokens.append( "$%s(%s)" % (token.getType(), token.getText()) )
+ self.popState()
+ return "%d: %s" % (self.index,string.join( next_tokens,' ' ) )
+
+ def printDebugMessage( self, message, new_level = None ):
+ ## margin = ''
+ ## if new_level == None:
+ ## margin = '|' * self.parser_depth
+ ## else:
+ ## if self.parser_depth > 0:
+ ## margin = '|' * (self.parser_depth-1)
+ ## margin += "+"
+ margin = '.' * self.parser_depth
+ print margin + '>' + message
+
class Parser:
""" Top-level class for all sub-parser.
***************
*** 341,346 ****
to set the name of the node returned by parser.
"""
! def __init__( self, name = None ):
! self.action_on_match = None # not use, use a decoarator parser instead (ActionParser)
self.name = name
--- 399,403 ----
to set the name of the node returned by parser.
"""
! def __init__( self, name = '' ):
self.name = name
***************
*** 348,352 ****
self.name = name
! def getName( self, name ):
return self.name
--- 405,409 ----
self.name = name
! def getName( self ):
return self.name
***************
*** 369,380 ****
result_value is the node representing the matched parse tree.
"""
! if self.action_on_match:
! result_value = self.action_on_match( result_value )
return Match( True, result_value )
! def notMatched( self ):
""" Called on parse failure to create the Match object."""
return Match( False )
class AlternativeParser(Parser):
def __init__( self, alt1, alt2 ):
--- 426,446 ----
result_value is the node representing the matched parse tree.
"""
! scanner.exitParser( self, "MATCH" )
return Match( True, result_value )
! def notMatched( self, scanner ):
""" Called on parse failure to create the Match object."""
+ scanner.exitParser( self, "FAIL" )
return Match( False )
+ def useShortDebugTrace( self ):
+ return False
+
+ def getDebugInfo( self ):
+ return ""
+
+ def isActionParser(self):
+ return False
+
class AlternativeParser(Parser):
def __init__( self, alt1, alt2 ):
***************
*** 383,386 ****
--- 449,453 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
for alternative in self.alternatives:
scanner.pushState()
***************
*** 390,394 ****
return self.matched( scanner, result.getValue() )
scanner.popState()
! return self.notMatched()
def addAlternative( self, alt ):
--- 457,461 ----
return self.matched( scanner, result.getValue() )
scanner.popState()
! return self.notMatched( scanner )
def addAlternative( self, alt ):
***************
*** 409,417 ****
def parse( self, scanner ):
results = Node( self.node_name )
for subparser in self.subparsers:
result = subparser.parse( scanner )
if not result:
! return self.notMatched()
results.addChild( result.getValue() )
return self.matched( scanner, results )
--- 476,485 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
results = Node( self.node_name )
for subparser in self.subparsers:
result = subparser.parse( scanner )
if not result:
! return self.notMatched( scanner )
results.addChild( result.getValue() )
return self.matched( scanner, results )
***************
*** 433,439 ****
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != SYMBOL or token.getText() != self.text:
! return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
--- 501,508 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
token = scanner.getNextToken()
if token.getType() != SYMBOL or token.getText() != self.text:
! return self.notMatched( scanner )
return self.matched( scanner, TokenNode( token ) )
***************
*** 441,444 ****
--- 510,519 ----
return '"%s"' % self.text
+ def useShortDebugTrace( self ):
+ return True
+
+ def getDebugInfo( self ):
+ return self.text
+
class ActionParser(Parser):
def __init__( self, parser, action ):
***************
*** 447,459 ****
self.action = action
def parse( self, scanner ):
result = self.parser.parse( scanner )
if result:
return self.action( result.getValue() )
! return self.notMatched()
def __str__( self ):
return "(%s)[action]" % str(self.parser)
class Action: # matched & notMatched really need to be moved somewhere else (free function ?)
def matched( self, node ):
--- 522,543 ----
self.action = action
+ def setName( self, name ):
+ Parser.setName( self, name )
+ if self.parser.getName() == '':
+ self.parser.setName( name )
+
def parse( self, scanner ):
+ scanner.enterParser( self )
result = self.parser.parse( scanner )
if result:
return self.action( result.getValue() )
! return self.notMatched( scanner )
def __str__( self ):
return "(%s)[action]" % str(self.parser)
+ def isActionParser(self):
+ return True
+
class Action: # matched & notMatched really need to be moved somewhere else (free function ?)
def matched( self, node ):
***************
*** 513,516 ****
--- 597,601 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
scanner.pushState()
result = self.optional_parser.parse( scanner )
***************
*** 531,534 ****
--- 616,620 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
results = Node( '?' )
repeat_count = 0
***************
*** 543,547 ****
if repeat_count >= self.min_count:
return self.matched( scanner, results )
! return self.notMatched()
def __str__( self ):
--- 629,633 ----
if repeat_count >= self.min_count:
return self.matched( scanner, results )
! return self.notMatched( scanner )
def __str__( self ):
***************
*** 558,561 ****
--- 644,648 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
results = Node( '?' )
while True:
***************
*** 577,583 ****
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != self.type:
! return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
--- 664,671 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
token = scanner.getNextToken()
if token.getType() != self.type:
! return self.notMatched( scanner )
return self.matched( scanner, TokenNode( token ) )
***************
*** 590,596 ****
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != INTEGER or token.getText() != '0':
! return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
--- 678,685 ----
def parse( self, scanner ):
+ scanner.enterParser( self )
token = scanner.getNextToken()
if token.getType() != INTEGER or token.getText() != '0':
! return self.notMatched( scanner )
return self.matched( scanner, TokenNode( token ) )
***************
*** 1064,1065 ****
--- 1153,1258 ----
function_try_block_p.setParser( symbol_p('try') >> maybe_p(ctor_initializer_p) >> function_body_p >> handler_seq_p )
+
+
+ # #################################### Debug information for parser ###################
+ # generated using generateparserdebuginfo.py (Notes: could this be done dynamically ?)
+
+ abstract_declarator_p.setName( 'abstract_declarator_p' )
+ access_level_spec_p.setName( 'access_level_spec_p' )
+ access_specifier_p.setName( 'access_specifier_p' )
+ array_type_p.setName( 'array_type_p' )
+ asm_definition_p.setName( 'asm_definition_p' )
+ asm_keyword_p.setName( 'asm_keyword_p' )
+ assignment_expression_p.setName( 'assignment_expression_p' )
+ attribut_decl_p.setName( 'attribut_decl_p' )
+ attribut_modifiers_p.setName( 'attribut_modifiers_p' )
+ base_clause_p.setName( 'base_clause_p' )
+ base_specifier_p.setName( 'base_specifier_p' )
+ basic_type_spec_p.setName( 'basic_type_spec_p' )
+ block_declaration_p.setName( 'block_declaration_p' )
+ block_statement_p.setName( 'block_statement_p' )
+ class_body_statements_p.setName( 'class_body_statements_p' )
+ class_decl_p.setName( 'class_decl_p' )
+ class_head_name_p.setName( 'class_head_name_p' )
+ class_head_p.setName( 'class_head_p' )
+ class_inheritance_and_body_decl_p.setName( 'class_inheritance_and_body_decl_p' )
+ class_inheritance_p.setName( 'class_inheritance_p' )
+ class_key_p.setName( 'class_key_p' )
+ class_name_p.setName( 'class_name_p' )
+ class_specialization_p.setName( 'class_specialization_p' )
+ class_specifier_p.setName( 'class_specifier_p' )
+ compound_statement_p.setName( 'compound_statement_p' )
+ constant_expression_p.setName( 'constant_expression_p' )
+ constant_initializer_p.setName( 'constant_initializer_p' )
+ ctor_initializer_p.setName( 'ctor_initializer_p' )
+ cv_modifier_p.setName( 'cv_modifier_p' )
+ cv_qualifier_p.setName( 'cv_qualifier_p' )
+ cv_qualifier_seq_p.setName( 'cv_qualifier_seq_p' )
+ decl_specifier_p.setName( 'decl_specifier_p' )
+ decl_specifier_seq_p.setName( 'decl_specifier_seq_p' )
+ declaration_p.setName( 'declaration_p' )
+ declaration_seq_p.setName( 'declaration_seq_p' )
+ declarator_p.setName( 'declarator_p' )
+ direct_abstract_declarator_p.setName( 'direct_abstract_declarator_p' )
+ direct_declarator_p.setName( 'direct_declarator_p' )
+ elaborated_type_specifier_p.setName( 'elaborated_type_specifier_p' )
+ end_p.setName( 'end_p' )
+ enum_specifier_p.setName( 'enum_specifier_p' )
+ enumerator_definition_p.setName( 'enumerator_definition_p' )
+ enumerator_p.setName( 'enumerator_p' )
+ eos_p.setName( 'eos_p' )
+ exception_declaration_p.setName( 'exception_declaration_p' )
+ exception_specification_p.setName( 'exception_specification_p' )
+ expression_list_p.setName( 'expression_list_p' )
+ friend_decl_p.setName( 'friend_decl_p' )
+ function_body_p.setName( 'function_body_p' )
+ function_decl_p.setName( 'function_decl_p' )
+ function_definition_p.setName( 'function_definition_p' )
+ function_definition_start_p.setName( 'function_definition_start_p' )
+ function_specifier_p.setName( 'function_specifier_p' )
+ function_try_block_p.setName( 'function_try_block_p' )
+ handler_p.setName( 'handler_p' )
+ handler_seq_p.setName( 'handler_seq_p' )
+ id_p.setName( 'id_p' )
+ init_declarator_list_p.setName( 'init_declarator_list_p' )
+ init_declarator_p.setName( 'init_declarator_p' )
+ initializer_clause_p.setName( 'initializer_clause_p' )
+ initializer_list_p.setName( 'initializer_list_p' )
+ initializer_p.setName( 'initializer_p' )
+ linkage_specification_begin_p.setName( 'linkage_specification_begin_p' )
+ linkage_specification_p.setName( 'linkage_specification_p' )
+ mem_initializer_p.setName( 'mem_initializer_p' )
+ member_declaration_p.setName( 'member_declaration_p' )
+ member_declarator_list_p.setName( 'member_declarator_list_p' )
+ member_declarator_p.setName( 'member_declarator_p' )
+ member_function_modifiers_p.setName( 'member_function_modifiers_p' )
+ member_initializers_p.setName( 'member_initializers_p' )
+ member_specification_p.setName( 'member_specification_p' )
+ named_namespace_definition_p.setName( 'named_namespace_definition_p' )
+ namespace_alias_definition_p.setName( 'namespace_alias_definition_p' )
+ namespace_body_p.setName( 'namespace_body_p' )
+ namespace_definition_p.setName( 'namespace_definition_p' )
+ nested_id_p.setName( 'nested_id_p' )
+ null_statement_p.setName( 'null_statement_p' )
+ opt_parameter_decl_assign_p.setName( 'opt_parameter_decl_assign_p' )
+ optional_initializer_p.setName( 'optional_initializer_p' )
+ parameter_declaration_clause_p.setName( 'parameter_declaration_clause_p' )
+ parameter_declaration_p.setName( 'parameter_declaration_p' )
+ ptr_operator_p.setName( 'ptr_operator_p' )
+ pure_specifier_p.setName( 'pure_specifier_p' )
+ qualified_namespace_specifier_p.setName( 'qualified_namespace_specifier_p' )
+ simple_declaration_p.setName( 'simple_declaration_p' )
+ simple_type_specifier_p.setName( 'simple_type_specifier_p' )
+ statements_p.setName( 'statements_p' )
+ storage_class_specifier_p.setName( 'storage_class_specifier_p' )
+ template_instantiation_param_p.setName( 'template_instantiation_param_p' )
+ throw_expression_p.setName( 'throw_expression_p' )
+ try_block_p.setName( 'try_block_p' )
+ type_id_list_p.setName( 'type_id_list_p' )
+ type_p.setName( 'type_p' )
+ type_specifier_p.setName( 'type_specifier_p' )
+ type_specifier_seq_p.setName( 'type_specifier_seq_p' )
+ type_suffix_p.setName( 'type_suffix_p' )
+ unnamed_namespace_definition_p.setName( 'unnamed_namespace_definition_p' )
+ using_declaration_p.setName( 'using_declaration_p' )
+ using_directive_p.setName( 'using_directive_p' )
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** parsertest.py 2 Aug 2003 18:33:53 -0000 1.4
--- parsertest.py 2 Aug 2003 21:29:08 -0000 1.5
***************
*** 134,143 ****
self.checkParser( "otot", proxy_test_p )
- ## self.checkParser( "namespace { } ", parser.namespace_definition_p )
- ## self.checkParser( "namespace MyWork { } ", parser.namespace_definition_p )
self.checkParser( "namespace fs = boost::filesystem;", parser.namespace_alias_definition_p )
self.checkParser( "using typename boost::filesystem::path;", parser.using_declaration_p )
self.checkParser( "using boost::filesystem::path;", parser.using_declaration_p )
self.checkParser( "using namespace boost::filesystem;", parser.using_directive_p )
# self.checkParser( 'extern "C" { }', parser.declaration_p )
--- 134,143 ----
self.checkParser( "otot", proxy_test_p )
self.checkParser( "namespace fs = boost::filesystem;", parser.namespace_alias_definition_p )
self.checkParser( "using typename boost::filesystem::path;", parser.using_declaration_p )
self.checkParser( "using boost::filesystem::path;", parser.using_declaration_p )
self.checkParser( "using namespace boost::filesystem;", parser.using_directive_p )
+ # self.checkParser( "namespace { } ", parser.namespace_definition_p )
+ ## self.checkParser( "namespace MyWork { } ", parser.namespace_definition_p )
# self.checkParser( 'extern "C" { }', parser.declaration_p )
|
|
From: <bl...@us...> - 2003-08-02 18:33:56
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv28200/src/pyrfta/test/rfta
Modified Files:
parser.py parsertest.py
Log Message:
* added massive amount of grammar for declaration, declarators, class & exception
* added ProxyParser to allow forward declaration of parser (allow recursion)
* namespace parsing is now failing (infinite recursion). => need to add a debug facility to the parser.
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** parser.py 1 Aug 2003 18:53:44 -0000 1.4
--- parser.py 2 Aug 2003 18:33:53 -0000 1.5
***************
*** 341,346 ****
to set the name of the node returned by parser.
"""
! def __init__( self ):
self.action_on_match = None # not use, use a decoarator parser instead (ActionParser)
def parse( self, scanner ):
--- 341,353 ----
to set the name of the node returned by parser.
"""
! def __init__( self, name = None ):
self.action_on_match = None # not use, use a decoarator parser instead (ActionParser)
+ self.name = name
+
+ def setName( self, name ):
+ self.name = name
+
+ def getName( self, name ):
+ return self.name
def parse( self, scanner ):
***************
*** 356,360 ****
The method matched() & notMatched() should be called to created the Match object.
"""
! pass
def matched( self, scanner, result_value ):
--- 363,367 ----
The method matched() & notMatched() should be called to created the Match object.
"""
! raise "Parser.parse() should have been implemented by the derived Parser"
def matched( self, scanner, result_value ):
***************
*** 545,549 ****
return 'repeat(%d,%s)' % (self.min_count,str(self.parser))
! class AnyUntil(Parser):
def __init__( self, until_parser ):
Parser.__init__( self )
--- 552,556 ----
return 'repeat(%d,%s)' % (self.min_count,str(self.parser))
! class AnyUntilParser(Parser):
def __init__( self, until_parser ):
Parser.__init__( self )
***************
*** 591,594 ****
--- 598,612 ----
return '0' % str(self.type)
+ class ProxyParser(Parser):
+ def __init__( self ):
+ Parser.__init__( self )
+ self.parser = None
+
+ def parse( self, scanner ):
+ return self.parser.parse( scanner )
+
+ def setParser( self, parser ):
+ self.parser = parser
+
class InlineChildAction(Action):
def __init__( self, node_index ):
***************
*** 662,665 ****
--- 680,686 ----
return RemoveSymbolAction()
+ def terminal_p( token_type ):
+ return TerminalParser( token_type )
+
end_p = TerminalParser( END_STREAM )
***************
*** 687,691 ****
def anyuntil_p( parser ):
""" Matchs any tokens until parser returns a match."""
! return AnyUntil( make_p( parser ) )
id_p = TerminalParser( IDENTIFIER )
--- 708,715 ----
def anyuntil_p( parser ):
""" Matchs any tokens until parser returns a match."""
! return AnyUntilParser( make_p( parser ) )
!
! def proxy_p():
! return ProxyParser()
id_p = TerminalParser( IDENTIFIER )
***************
*** 774,775 ****
--- 798,1065 ----
| class_decl_p
+ # ############################## A.5: Expressions ############################
+ # A.5, C++PL3 p.798-802
+ constant_expression_p = terminal_p(INTEGER) | terminal_p(CHAR) # dummy
+
+ # dummy
+ assignment_expression_p = constant_expression_p
+
+ ##assignment_expression_p = proxy_p()
+ ##
+ ##assignment_expression_p.setParser( conditional_expression_p \
+ ## | ( logical_or_expression_p >> assignement_operator_p >> assignment_expression_p ) \
+ ## | throw_expression_p )
+
+ # dummy
+ expression_list_p = constant_expression_p
+
+ # ############################## A.6: Statements ############################
+
+ #dummy
+ compound_statement_p = symbol_p('{') >> '}'
+
+ # ############################## A.7.1: Declarators ############################
+ # A.7.1, C++PL3 p.807-808
+
+ class_key_p = proxy_p()
+ class_specifier_p = proxy_p()
+ ctor_initializer_p = proxy_p()
+ declarator_p = proxy_p()
+ decl_specifier_p = proxy_p() # forward declaration for recursion
+ exception_specification_p = proxy_p()
+ function_try_block_p = proxy_p()
+ ptr_operator_p = proxy_p()
+ abstract_declarator_p = proxy_p()
+ direct_abstract_declarator_p = proxy_p()
+ parameter_declaration_clause_p = proxy_p()
+ initializer_clause_p = proxy_p()
+ type_specifier_p = proxy_p()
+
+ cv_qualifier_p = symbol_p('const') | 'volatile'
+ cv_qualifier_seq_p = repeat_p( 1, cv_qualifier_p )
+
+ decl_specifier_seq_p = repeat_p( 1, decl_specifier_p )
+
+ function_body_p = compound_statement_p
+
+ function_definition_start_p = maybe_p(decl_specifier_seq_p) >> declarator_p
+ function_definition_p = function_definition_start_p \
+ >> ( (maybe_p(ctor_initializer_p) >> function_body_p) \
+ | function_try_block_p )
+
+
+ direct_abstract_declarator_p.setParser( (maybe_p( direct_abstract_declarator_p ) >> '(' >> parameter_declaration_clause_p >> ')' \
+ >> maybe_p( cv_qualifier_seq_p ) >> maybe_p( exception_specification_p ) ) \
+ | (maybe_p( direct_abstract_declarator_p ) >> '[' >> constant_expression_p >> ']') \
+ | (symbol_p('(') >> direct_abstract_declarator_p >> ')') )
+
+ abstract_declarator_p.setParser( (ptr_operator_p >> maybe_p( abstract_declarator_p )) \
+ | direct_abstract_declarator_p )
+
+ opt_parameter_decl_assign_p = maybe_p( symbol_p('=') >> assignment_expression_p )
+
+ parameter_declaration_p = (decl_specifier_seq_p >> declarator_p >> opt_parameter_decl_assign_p ) \
+ | (decl_specifier_seq_p >> abstract_declarator_p >> opt_parameter_decl_assign_p )
+
+ parameter_declaration_clause_p.setParser( symbol_p('...') \
+ | (maybe_p( list_p( parameter_declaration_p, ',' ) ) \
+ >> maybe_p( symbol_p(',') >> symbol_p('...') ) ) )
+
+ # Notes: drastically simplified. Is it correct ?
+ declarator_id = nested_id_p
+
+ ptr_operator_p.setParser( (symbol_p('*') >> maybe_p( cv_qualifier_seq_p )) \
+ | symbol_p('&') \
+ | (nested_id_p >> '*' >> maybe_p( cv_qualifier_seq_p ) ) )
+
+ direct_declarator_p = proxy_p()
+ direct_declarator_p.setParser( declarator_id \
+ | (direct_declarator_p >> '(' >> parameter_declaration_clause_p >>')' \
+ >> maybe_p( cv_qualifier_seq_p ) >> maybe_p( exception_specification_p ) ) \
+ | (direct_declarator_p >> '[' >> maybe_p(constant_expression_p) >> ']') \
+ | ( symbol_p('(') >> declarator_p >> ')') )
+
+ declarator_p.setParser( direct_declarator_p \
+ | (ptr_operator_p >> declarator_p) )
+
+ initializer_list_p = list_p( initializer_clause_p, ',' )
+
+ initializer_clause_p.setParser( assignment_expression_p \
+ | (symbol_p('{') >> initializer_list_p >> maybe_p( ',' ) >> '}') \
+ | (symbol_p('{') >> '}') )
+
+ initializer_p = initializer_clause_p \
+ | (symbol_p('(') >> expression_list_p >> ')')
+
+ init_declarator_p = declarator_p >> maybe_p( initializer_p )
+
+ init_declarator_list_p = list_p( init_declarator_p, ',' )
+
+ type_specifier_seq_p = repeat_p( 1, type_specifier_p )
+
+ # ############################## A.7: Declaration ############################
+ declaration_p = proxy_p() # forward declaration for recursion
+
+ declaration_seq_p = repeat_p( 1, declaration_p )
+
+ # A.7, C++PL3 p.805-806
+ # => parse namespace declaration and namespace alias declaration
+ # Notes: OK. No parsing issue
+ namespace_body_p = maybe_p( declaration_seq_p )['namespace_body']
+ qualified_namespace_specifier_p = nested_id_p
+ named_namespace_definition_p = symbol_p('namespace') >> id_p >> '{' >> namespace_body_p >> '}'
+ unnamed_namespace_definition_p = (symbol_p('namespace') >> '{' >> namespace_body_p >> '}')
+ namespace_alias_definition_p = (symbol_p('namespace') >> id_p >> '=' \
+ >> qualified_namespace_specifier_p >> eos_p)['namespace_alias_def'][remove_symbol_a()]
+ namespace_definition_p = ( named_namespace_definition_p['named_namespace_def'] \
+ | unnamed_namespace_definition_p['unnamed_namespace_def'] )[remove_symbol_a()]
+
+ # A.7, C++PL3 p.806
+ # => parse using declaration and directive
+ # Notes: OK. No parsing issue
+ using_declaration_p = ( ( (symbol_p('using') >> 'typename' >> nested_id_p)['using_typename_decl'] \
+ | (symbol_p('using') >> nested_id_p)['using_decl'] ) \
+ >> eos_p )
+
+ using_directive_p = symbol_p('using') >> 'namespace' >> nested_id_p >> eos_p
+
+ # A.7, C++PL3 p.806
+ # => parse asm definition
+ asm_keyword_p = symbol_p('asm') \
+ | '__asm' # Microsoft Visual C++
+
+ asm_definition_p = asm_keyword_p >> '(' >> terminal_p(STRING) >> ')' >> eos_p
+
+ # A.7, C++PL3 p.806
+ # => parse extern specifiction
+ linkage_specification_begin_p = symbol_p('extern') >> terminal_p(STRING)
+ linkage_specification_p = (linkage_specification_begin_p >> '{' >> maybe_p(declaration_seq_p) >> '}') \
+ | (linkage_specification_begin_p >> declaration_p)
+
+ # A.7, C++PL3 p.804-806
+ # => parse simple declaration
+ # Notes: parsing issue in:
+ # -simple_type_specifier_p
+ # -enumerator_definition_p (for constant expression that contains template with multiple parameters)
+
+ enumerator_p = id_p
+
+ enumerator_definition_p = enumerator_p \
+ | enumerator_p >> '=' >> constant_expression_p
+
+ enum_specifier_p = symbol_p('enum') >> maybe_p(id_p) >> '{' >> maybe_p(list_p(enumerator_definition_p, ',')) >> '}'
+
+ elaborated_type_specifier_p = class_key_p >> nested_id_p \
+ | (symbol_p('enum') >> nested_id_p) \
+ | (symbol_p('typename') >> nested_id_p) \
+ | (symbol_p('typename') >> nested_id_p >> 'template' >> id_p ) # need to check that one. Is it really a nest_id_p
+
+ # Notes: following case not handled:
+ # - opt(::) opt(nested-name-specifier) type-name
+ # - opt(::) opt(nested-name-specifier) opt(tempalte) template-id
+ simple_type_specifier_p = symbol_p('char') | 'wchar_t' | 'bool' | 'short' | 'int' | 'long' \
+ | 'signed' | 'unsigned' | 'float' | 'double' | 'void'
+
+ storage_class_specifier_p = symbol_p('auto') | 'register' | 'static' | 'extern' | 'mutable'
+ function_specifier_p = symbol_p('inline') | 'virtual' | 'explicit'
+ type_specifier_p.setParser( simple_type_specifier_p
+ | class_specifier_p
+ | enum_specifier_p
+ | elaborated_type_specifier_p
+ | cv_qualifier_p )
+
+ decl_specifier_p.setParser( storage_class_specifier_p \
+ | type_specifier_p \
+ | function_specifier_p \
+ | 'friend' \
+ | 'typedef' )
+
+ simple_declaration_p = repeat_p( 0, decl_specifier_p ) >> maybe_p( init_declarator_list_p ) >> eos_p
+
+ # A.7, C++PL3 p.804
+ block_declaration_p = asm_definition_p \
+ | namespace_alias_definition_p \
+ | using_declaration_p \
+ | using_directive_p \
+ | simple_declaration_p
+
+ declaration_p.setParser( block_declaration_p \
+ | linkage_specification_p \
+ | namespace_definition_p )
+ # | function_definition_p \
+ # | template_declaration_p \
+ # | explicit_instantiation_p \
+ # | explicit_specialization_p \
+
+
+ # ############################## A.8: Classes ############################
+ # A.8, C++PL3 p.808-809
+
+ class_name_p = id_p
+
+ constant_initializer_p = symbol_p('=') >> constant_expression_p
+
+ pure_specifier_p = symbol_p('=') >> ZeroParser()
+
+ member_declarator_p = (declarator_p >> maybe_p(pure_specifier_p)) \
+ | (declarator_p >> maybe_p(constant_initializer_p)) \
+ | (maybe_p(id_p) >> ':' >> constant_expression_p)
+
+ member_declarator_list_p = list_p( member_declarator_p, ',' )
+
+ #Notes: not included:
+ #- opt(::) nested-name-specifier opt(template) unqualified-id
+ member_declaration_p = (maybe_p( decl_specifier_seq_p ) >> maybe_p( member_declarator_list_p ) >> eos_p) \
+ | (function_definition_p >> maybe_p( eos_p )) \
+ | using_declaration_p
+ # | template_declaration_p
+
+ access_specifier_p = symbol_p("public") | "protected" | "private"
+
+ member_specification_p = proxy_p()
+ member_specification_p.setParser( repeat_p( 0,
+ (access_specifier_p >> ':') \
+ | member_declaration_p ) )
+
+ class_key_p.setParser( symbol_p('class') | 'struct' | 'union' )
+
+ base_specifier_p = maybe_p('virtual') >> maybe_p( access_specifier_p ) \
+ >> maybe_p('virtual') >> nested_id_p # not template support in class_name for now
+
+ base_clause_p = symbol_p(':') >> list_p( base_specifier_p, ',' )
+
+ class_head_name_p = nested_id_p >> maybe_p( symbol_p('template') >> id_p )
+
+ class_head_p = class_key_p >> maybe_p( class_head_name_p ) >> maybe_p( base_clause_p )
+
+ class_specifier_p.setParser( class_head_p >> '{' >> maybe_p( member_specification_p ) >> '}' )
+
+ # ############### A.8.2: Classes, Specials Member Functions ##################
+ # A.8.2, C++PL3 p.810
+
+ mem_initializer_id = nested_id_p
+
+ mem_initializer_p = mem_initializer_id >> '(' >> maybe_p( expression_list_p ) >> ')'
+
+ ctor_initializer_p.setParser( symbol_p( ':' ) >> list_p( mem_initializer_p, ',' ) )
+
+
+
+ # ############################## A.10: Exceptions ############################
+ # A.10, C++PL3 p.812
+
+ throw_expression_p = symbol_p('throw') >> maybe_p( assignment_expression_p )
+
+ exception_declaration_p = (type_specifier_seq_p >> maybe_p( declarator_p | abstract_declarator_p )) \
+ | '...'
+
+ handler_p = symbol_p('catch') >> '(' >> exception_declaration_p >> ')' >> compound_statement_p
+
+ handler_seq_p = repeat_p( 1, handler_p )
+
+ type_id_list_p = list_p( nested_id_p, ',' )
+ exception_specification_p.setParser( symbol_p( 'throw' ) >> '(' >> maybe_p( type_id_list_p ) >> ')' )
+
+ try_block_p = symbol_p('try') >> compound_statement_p >> handler_seq_p
+
+ function_try_block_p.setParser( symbol_p('try') >> maybe_p(ctor_initializer_p) >> function_body_p >> handler_seq_p )
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** parsertest.py 1 Aug 2003 08:25:05 -0000 1.3
--- parsertest.py 2 Aug 2003 18:33:53 -0000 1.4
***************
*** 130,133 ****
--- 130,146 ----
self.checkParser( "class MyClass : public MyParent, MyParent2 { };", parser.class_decl_p )
+ proxy_test_p = parser.proxy_p()
+ proxy_test_p.setParser( parser.id_p )
+ self.checkParser( "otot", proxy_test_p )
+
+ ## self.checkParser( "namespace { } ", parser.namespace_definition_p )
+ ## self.checkParser( "namespace MyWork { } ", parser.namespace_definition_p )
+ self.checkParser( "namespace fs = boost::filesystem;", parser.namespace_alias_definition_p )
+ self.checkParser( "using typename boost::filesystem::path;", parser.using_declaration_p )
+ self.checkParser( "using boost::filesystem::path;", parser.using_declaration_p )
+ self.checkParser( "using namespace boost::filesystem;", parser.using_directive_p )
+ # self.checkParser( 'extern "C" { }', parser.declaration_p )
+
+
def checkParser( self, text, tested_parser ):
tokens = self.scanner.tokenize( text )
|
|
From: <bl...@us...> - 2003-08-01 18:53:47
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv16704/src/pyrfta/test/rfta
Modified Files:
parser.py
Log Message:
* added action to remove symbol
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** parser.py 1 Aug 2003 08:25:05 -0000 1.3
--- parser.py 1 Aug 2003 18:53:44 -0000 1.4
***************
*** 512,516 ****
return self.matched( scanner, result.getValue() )
scanner.popState()
! return self.matched( scanner, None )
def __str__( self ):
--- 512,516 ----
return self.matched( scanner, result.getValue() )
scanner.popState()
! return self.matched( scanner, Node('missing') )
def __str__( self ):
***************
*** 622,625 ****
--- 622,636 ----
return self.subparser.parse( local_scanner )
+ class RemoveSymbolAction(Action):
+ def __call__( self, node ):
+ children = node.getChildren()
+ new_children = []
+ for child_node in children:
+ if child_node.isLeaf() and child_node.getToken().getType() == SYMBOL:
+ continue
+ new_children.append( child_node )
+ new_node = Node(node.getType(), new_children)
+ return self.matched( new_node )
+
def inline_child_at_a( node_index ):
return InlineChildAction( node_index )
***************
*** 648,651 ****
--- 659,665 ----
return SubParseAction( make_p( parser ) )
+ def remove_symbol_a():
+ return RemoveSymbolAction()
+
end_p = TerminalParser( END_STREAM )
***************
*** 678,683 ****
eos_p = symbol_p( ';' )
!
! nested_id_p = (maybe_p( "::" ) >> list_p( id_p, symbol_p('::') ))[inline_child_at_a(1)]['nested_id']
# use separate parser for inheritance => allow other alternative matching
--- 692,697 ----
eos_p = symbol_p( ';' )
! nested_id_p = (maybe_p( "::" ) >> \
! list_p( id_p, symbol_p('::') )[remove_symbol_a()])[inline_child_at_a(1)]['nested_id']
# use separate parser for inheritance => allow other alternative matching
|
|
From: <bl...@us...> - 2003-08-01 08:25:08
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv24537/rfta
Modified Files:
parser.py parsertest.py
Log Message:
* added subparser support
* parse class base classes declaration using a subparser
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** parser.py 31 Jul 2003 18:31:50 -0000 1.2
--- parser.py 1 Aug 2003 08:25:05 -0000 1.3
***************
*** 275,278 ****
--- 275,281 ----
return []
+ def getToken( self ):
+ return self.token
+
def isLeaf( self ):
return True
***************
*** 440,444 ****
result = self.parser.parse( scanner )
if result:
! return self.matched( scanner, self.action( result.getValue() ) )
return self.notMatched()
--- 443,447 ----
result = self.parser.parse( scanner )
if result:
! return self.action( result.getValue() )
return self.notMatched()
***************
*** 446,450 ****
return "(%s)[action]" % str(self.parser)
! class NameAction:
def __init__( self, name ):
self.name = name
--- 449,460 ----
return "(%s)[action]" % str(self.parser)
! class Action: # matched & notMatched really need to be moved somewhere else (free function ?)
! def matched( self, node ):
! return Match( True, node )
!
! def notMatched( self ):
! return Match( False )
!
! class NameAction(Action):
def __init__( self, name ):
self.name = name
***************
*** 452,456 ****
def __call__( self, node ):
node.setType( self.name )
! return node
def name_a( name ):
--- 462,466 ----
def __call__( self, node ):
node.setType( self.name )
! return self.matched( node )
def name_a( name ):
***************
*** 581,585 ****
return '0' % str(self.type)
! class InlineChildAction:
def __init__( self, node_index ):
self.node_index = node_index
--- 591,595 ----
return '0' % str(self.type)
! class InlineChildAction(Action):
def __init__( self, node_index ):
self.node_index = node_index
***************
*** 588,594 ****
inlined_node = node.getChildren()[ self.node_index ]
node.getChildren()[self.node_index : self.node_index+1] = inlined_node.getChildren()
! return node
! class InlineChildrenAction:
def __call__( self, node ):
new_children = []
--- 598,604 ----
inlined_node = node.getChildren()[ self.node_index ]
node.getChildren()[self.node_index : self.node_index+1] = inlined_node.getChildren()
! return self.matched( node )
! class InlineChildrenAction(Action):
def __call__( self, node ):
new_children = []
***************
*** 599,603 ****
new_children.extend( child.getChildren() )
new_node = Node(node.getType(), new_children)
! return new_node
def inline_child_at_a( node_index ):
--- 609,624 ----
new_children.extend( child.getChildren() )
new_node = Node(node.getType(), new_children)
! return self.matched( new_node )
!
! class SubParseAction(Action):
! def __init__( self, subparser ):
! self.subparser = subparser
!
! def __call__( self, node ):
! tokens = []
! for token_node in node.getChildren(): # all children must be TokenNode => need to test and dump an helpful message
! tokens.append( token_node.getToken() )
! local_scanner = TokenProvider( tokens )
! return self.subparser.parse( local_scanner )
def inline_child_at_a( node_index ):
***************
*** 624,627 ****
--- 645,651 ----
return InlineChildrenAction()
+ def subparse_a( parser ):
+ return SubParseAction( make_p( parser ) )
+
end_p = TerminalParser( END_STREAM )
***************
*** 709,713 ****
| friend_decl_p # enum, nested_class/struct, how ?
! class_inheritance_p = maybe_p( (symbol_p( ":" ) >> anyuntil_p( "{" ))['inheritance'] )
class_inheritance_and_body_decl_p = class_inheritance_p >> "{" >> maybe_p(class_body_statements_p) >> "}"
--- 733,745 ----
| friend_decl_p # enum, nested_class/struct, how ?
! # A.8.1, C++PL3 p. 810
! access_specifier_p = symbol_p('public') | 'protected' | 'private'
!
! base_specifier_p = maybe_p('virtual') >> maybe_p( access_specifier_p ) \
! >> maybe_p('virtual') >> nested_id_p # not template support in class_name for now
!
! base_clause_subp = list_p( base_specifier_p['base_class'], ',' )['inheritance'] >> end_p
!
! class_inheritance_p = maybe_p( (symbol_p( ":" ) >> anyuntil_p( "{" )[subparse_a( base_clause_subp )]) )
class_inheritance_and_body_decl_p = class_inheritance_p >> "{" >> maybe_p(class_body_statements_p) >> "}"
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** parsertest.py 31 Jul 2003 18:31:50 -0000 1.2
--- parsertest.py 1 Aug 2003 08:25:05 -0000 1.3
***************
*** 128,132 ****
self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
! self.checkParser( "class MyClass : public MyParent<MyClass,simple_traits> { };", parser.class_decl_p )
def checkParser( self, text, tested_parser ):
--- 128,132 ----
self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
! self.checkParser( "class MyClass : public MyParent, MyParent2 { };", parser.class_decl_p )
def checkParser( self, text, tested_parser ):
|
|
From: <bl...@us...> - 2003-07-31 18:31:53
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv11700/rfta
Modified Files:
parser.py parsertest.py
Log Message:
* fixed class body parsing
Index: parser.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parser.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** parser.py 31 Jul 2003 08:00:06 -0000 1.1
--- parser.py 31 Jul 2003 18:31:50 -0000 1.2
***************
*** 708,714 ****
| function_decl_p \
| friend_decl_p # enum, nested_class/struct, how ?
-
! class_inheritance_and_body_decl = symbol_p( ":" ) >> anyuntil_p( "{" ) >> class_body_statements_p >> "}"
template_instantiation_param_p3 = anyuntil_p( symbol_p(">") )
--- 708,715 ----
| function_decl_p \
| friend_decl_p # enum, nested_class/struct, how ?
! class_inheritance_p = maybe_p( (symbol_p( ":" ) >> anyuntil_p( "{" ))['inheritance'] )
!
! class_inheritance_and_body_decl_p = class_inheritance_p >> "{" >> maybe_p(class_body_statements_p) >> "}"
template_instantiation_param_p3 = anyuntil_p( symbol_p(">") )
***************
*** 720,724 ****
class_decl_p = (symbol_p("class") | "struct") >> nested_id_p >> maybe_p(class_specialization_p) \
! >> maybe_p( class_inheritance_and_body_decl ) >> eos_p
statements = end_p \
--- 721,725 ----
class_decl_p = (symbol_p("class") | "struct") >> nested_id_p >> maybe_p(class_specialization_p) \
! >> maybe_p( class_inheritance_and_body_decl_p ) >> eos_p
statements = end_p \
Index: parsertest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/parsertest.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** parsertest.py 31 Jul 2003 08:00:06 -0000 1.1
--- parsertest.py 31 Jul 2003 18:31:50 -0000 1.2
***************
*** 128,131 ****
--- 128,132 ----
self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
+ self.checkParser( "class MyClass : public MyParent<MyClass,simple_traits> { };", parser.class_decl_p )
def checkParser( self, text, tested_parser ):
***************
*** 133,137 ****
scanner = parser.TokenProvider( tokens )
match = tested_parser.parse( scanner )
! self.assert_( match, "nested_id_p should have matched source" )
remaining_tokens = []
while scanner.hasMoreToken() :
--- 134,138 ----
scanner = parser.TokenProvider( tokens )
match = tested_parser.parse( scanner )
! self.assert_( match, "parser should have matched source" )
remaining_tokens = []
while scanner.hasMoreToken() :
|
|
From: <bl...@us...> - 2003-07-31 08:21:51
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv26696
Modified Files:
alltests.py codeanalysis.py codeanalysistest.py
Log Message:
* added more sophisticated code analysis (still some failing test)
* disabled code analysis test as focus is on the parser for the current time.
Index: alltests.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/alltests.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -C2 -d -r1.1 -r1.2
*** alltests.py 29 May 2003 14:05:56 -0000 1.1
--- alltests.py 31 Jul 2003 08:21:48 -0000 1.2
***************
*** 10,18 ****
import unittest
import rfta.codeanalysistest
def suite():
# modules_to_test = ('rfta.codeanalysistest') # and so on
alltests = unittest.TestSuite()
! alltests.addTest(unittest.findTestCases(rfta.codeanalysistest))
# for module in map(__import__, modules_to_test):
# alltests.addTest(unittest.findTestCases(module))
--- 10,20 ----
import unittest
import rfta.codeanalysistest
+ import rfta.parsertest
def suite():
# modules_to_test = ('rfta.codeanalysistest') # and so on
alltests = unittest.TestSuite()
! # alltests.addTest(unittest.findTestCases(rfta.codeanalysistest))
! alltests.addTest(unittest.findTestCases(rfta.parsertest))
# for module in map(__import__, modules_to_test):
# alltests.addTest(unittest.findTestCases(module))
Index: codeanalysis.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/codeanalysis.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** codeanalysis.py 30 May 2003 07:31:24 -0000 1.4
--- codeanalysis.py 31 Jul 2003 08:21:48 -0000 1.5
***************
*** 4,7 ****
--- 4,10 ----
class LocalVariableDeclarator(rfta.StatementVisitor,rfta.ExpressionVisitor):
+ """Declare local variables and resolve all unqualified identifiers
+ found while walking the statements and expressions of the specified
+ function body"""
def __init__( self, resolver ):
rfta.StatementVisitor.__init__( self )
***************
*** 23,27 ****
def visitDeclaratorExpression( self, expression ):
for declarator in expression.getDeclarators():
! self.resolver.declareVariable( declarator.name.identifier, expression )
def visitForStatement( self, statement ):
--- 26,30 ----
def visitDeclaratorExpression( self, expression ):
for declarator in expression.getDeclarators():
! self.resolver.declareVariable( declarator.name, expression )
def visitForStatement( self, statement ):
***************
*** 47,51 ****
self.resolver.exitScope()
! class Identifier:
def __init__( self, identifier ):
self.identifier = identifier
--- 50,54 ----
self.resolver.exitScope()
! class IdentifierRef:
def __init__( self, identifier ):
self.identifier = identifier
***************
*** 55,64 ****
if pos == -1:
return self.identifier
! return self.identifier[pos:]
! IDTYPE_LOCALVARIABLE = 1
! IDTYPE_IMPORTEDID = 2
class Scope:
def __init__( self, parent_scope ):
self.children = []
--- 58,76 ----
if pos == -1:
return self.identifier
! return self.identifier[pos+2:]
! IDTYPE_LOCALVARIABLE = "local_variable"
! IDTYPE_IMPORTEDID = "using_decl"
! IDTYPE_PARAMETER = "function_parameter"
! IDTYPE_METHOD = "class_method"
! IDTYPE_ATTRIBUT = "class_attribut"
! IDTYPE_NESTED_TYPE = "class_nested_type"
class Scope:
+ """Represents a specified scope used to resolve identifier.
+
+ Scopes form a hierarchy. When a identifier is not resolved
+ by a given scope, the identifier resolution is forwarded to
+ the parent scope."""
def __init__( self, parent_scope ):
self.children = []
***************
*** 68,72 ****
def resolve( self, identifier ):
! return None
def parentResolve( self, identifier ):
--- 80,90 ----
def resolve( self, identifier ):
! """Resolve the specified identifier. Return a tuple of the
! form (identifier_type, declaration_element). None is returned
! if the identifier was not resolved.
!
! The specified identifier may contains nested scope (id::id::id).
! """
! return self.parentResolve()
def parentResolve( self, identifier ):
***************
*** 75,90 ****
return None
class FunctionScope(Scope):
def __init__( self, function ):
Scope.__init__( self, None )
self.function = function
class LocalVariableScope(Scope):
def __init__( self, parent_scope ):
Scope.__init__( self, parent_scope )
self.declared = {}
! def declare( self, variableName, declExpr ):
! self.declared[variableName] = declExpr
def declareLocalVariable( self, variableName ):
--- 93,159 ----
return None
+ class UsingNamespaceScope(Scope):
+ """Scope of a using directive (using namespace std)."""
+ def __init__( self, using_directive_expr, parent_scope ):
+ Scope.__init__( self, parent_scope )
+ self.using_directive = using_directive_expr
+
+ def resolve( self, identifier ):
+ # ambiguity error if both UsingNamespaceScope and parent can resolve
+ with_using = self.manager.resolve( self.using_directive.namespace_name.name + "::" + identifier )
+ if not with_using:
+ return self.parentResolve( identifier )
+
+ class ClassImplScope(Scope):
+ def __init__( self, class_decl, parent_scope ):
+ Scope.__init__( self, parent_scope )
+ self.class_decl = class_decl
+
+ def resolve( self, identifier ):
+ for method in self.class_decl.getMethods():
+ if method.name.name == identifier:
+ return (IDTYPE_METHOD, method)
+ for attribut in self.class_decl.getAttributs():
+ if attribut.name.name == identifier:
+ return (IDTYPE_ATTRIBUT, attribut )
+ for type in self.class_decl.getNestedTypes():
+ if type.name.name == identifier:
+ return (IDTYPE_NESTED_TYPE, type )
+
class FunctionScope(Scope):
+ """The scope of a function declaration.
+
+ Resolve identifier matching the function parameters."""
def __init__( self, function ):
Scope.__init__( self, None )
self.function = function
+ class RealFunctionScope(Scope):
+ """The scope of a function declaration.
+
+ Resolve identifier matching the function parameters."""
+ def __init__( self, function ):
+ Scope.__init__( self, None )
+ self.function = function
+
+ def resolve( self, identifier ):
+ for parameter in self.function.getParameters():
+ if parameter.name.name == str(identifier):
+ return (IDTYPE_PARAMETER, parameter )
+ return self.parentResolve( identifier )
+
class LocalVariableScope(Scope):
+ """The scope of some local variables declaration.
+
+ Resolve identifier matching the locale variable declared in
+ this scope."""
def __init__( self, parent_scope ):
Scope.__init__( self, parent_scope )
self.declared = {}
! def declare( self, variable_name, decl_expr ):
! """Declare a local variable.
! declExpr must be the DeclarationExpression."""
! self.declared[variable_name] = decl_expr
def declareLocalVariable( self, variableName ):
***************
*** 97,127 ****
class UsingDeclScope(Scope):
! def __init__( self, parent_scope ):
Scope.__init__( self, parent_scope )
! self.imported = {}
!
! def importIdentifier( self, identifier ):
! imported[ Identifier(identifier).name() ] = identifier
def resolve( self, identifier ):
! if declareLocalVariable( self, identifier ):
! return (IDTYPE_IMPORTEDID, imported[identifier])
return self.parentResolve( identifier )
class LocalVariableResolver:
def __init__( self, function_scope ):
self.current_scope = function_scope
self.standard_for_scope = True
- self.scope_depth = 0
self.resolved = {}
! def resolveVariable( self, variableName, variable_name_element ):
! variableDecl = self.current_scope.resolve( variableName )
! if variableDecl:
! self.resolved[ variable_name_element ] = variableDecl
! def declareVariable( self, variableName, declExpr ):
! assert( self.scope_depth > 0 )
! self.current_scope.declare( variableName, declExpr )
def findVariable( self, variable_name_element ):
--- 166,204 ----
class UsingDeclScope(Scope):
! """Scope of using declaration (using BaseClass::function).
!
! Resolve the imported identifier."""
! def __init__( self, parent_scope, imported_identifier, using_expr ):
Scope.__init__( self, parent_scope )
! self.identifier = imported_identifier
! self.using_expr = using_expr
def resolve( self, identifier ):
! if str(identifier) == self.identifier.name():
! return (IDTYPE_IMPORTEDID, self.using_expr)
return self.parentResolve( identifier )
class LocalVariableResolver:
+ """Builder that creates the scope hierarchy while the code model
+ is traversed. Maintains the list of resolved identifiers."""
def __init__( self, function_scope ):
self.current_scope = function_scope
self.standard_for_scope = True
self.resolved = {}
+ self.exit_scopes = []
! def resolveVariable( self, variable_name, variable_name_element ):
! variable_decl = self.current_scope.resolve( variable_name )
! if variable_decl:
! self.resolved[ variable_name_element ] = variable_decl
! def declareVariable( self, variable_name, decl_expr ):
! assert( len(self.exit_scopes) > 0 )
! self.current_scope.declare( variable_name.identifier, decl_expr )
! self.resolved[ variable_name ] = decl_expr
!
! def usingDeclaration( self, identifier, usingExpr ):
! using_scope = UsingDeclScope( self.current_scope, identifier, usingExpr )
! self.current_scope = LocalVariableScope( using_scope )
def findVariable( self, variable_name_element ):
***************
*** 129,139 ****
def enterScope( self ):
self.current_scope = LocalVariableScope( self.current_scope )
- self.scope_depth += 1
def exitScope( self ):
! assert( self.scope_depth > 0 )
! self.scope_depth -= 1
! self.current_scope = self.current_scope.parent_scope
def enterForScope( self ):
--- 206,215 ----
def enterScope( self ):
+ self.exit_scopes.append( self.current_scope )
self.current_scope = LocalVariableScope( self.current_scope )
def exitScope( self ):
! assert( len(self.exit_scopes) > 0 )
! self.current_scope = self.exit_scopes.pop()
def enterForScope( self ):
***************
*** 144,148 ****
if self.standard_for_scope:
self.exitScope()
-
class Analyser:
--- 220,223 ----
Index: codeanalysistest.py
===================================================================
RCS file: /cvsroot/cpptool/rfta/src/pyrfta/test/rfta/codeanalysistest.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -C2 -d -r1.4 -r1.5
*** codeanalysistest.py 30 May 2003 07:31:25 -0000 1.4
--- codeanalysistest.py 31 Jul 2003 08:21:48 -0000 1.5
***************
*** 87,91 ****
actual_variables = []
for variable_decl in self.resolver.declareVariable.argsPassed:
! actual_variables.append( variable_decl[0] )
self.assertEqual( actual_variables, expected_variables )
--- 87,91 ----
actual_variables = []
for variable_decl in self.resolver.declareVariable.argsPassed:
! actual_variables.append( variable_decl[0].identifier )
self.assertEqual( actual_variables, expected_variables )
***************
*** 110,114 ****
self.declarator.declare( self.body )
self.assertEqual( self.resolver.called, [ "enterScope", "declareVariable", "declareVariable", "exitScope" ] )
! self.checkDeclaredVariable( ["var1", "var2" ] )
def testWhileScope( self ):
--- 110,114 ----
self.declarator.declare( self.body )
self.assertEqual( self.resolver.called, [ "enterScope", "declareVariable", "declareVariable", "exitScope" ] )
! self.checkDeclaredVariable( ['var1', 'var2'] )
def testWhileScope( self ):
***************
*** 158,162 ****
def setUp( self ):
self.function = self.makeQuickFunctionDecl( "void", "f" )
! self.resolver = LocalVariableResolver( FunctionScope( self.function ) )
self.declExpr1 = self.makeVariableDeclExpr( [("var1", "1"), ("var2","b"), ("var3","c")] )
self.declExpr1b = self.makeVariableDeclExpr( [("var1", "a"), ("var2","z")] )
--- 158,163 ----
def setUp( self ):
self.function = self.makeQuickFunctionDecl( "void", "f" )
! self.function_scope = FunctionScope( self.function )
! self.resolver = LocalVariableResolver( self.function_scope )
self.declExpr1 = self.makeVariableDeclExpr( [("var1", "1"), ("var2","b"), ("var3","c")] )
self.declExpr1b = self.makeVariableDeclExpr( [("var1", "a"), ("var2","z")] )
***************
*** 165,170 ****
self.declExpr3 = self.makeVariableDeclExpr( [("a", "1"), ("b","b"), ("c","c")] )
self.declExpr3b = self.makeVariableDeclExpr( [("a", "1"), ("b","b")] )
self.identifiers = {}
! for name in [ "var1", "var2", "var3", "x", "y", "z", "a", "b", "c" ]:
for count in range(1,10):
variable_name = name + "." + str(count)
--- 166,172 ----
self.declExpr3 = self.makeVariableDeclExpr( [("a", "1"), ("b","b"), ("c","c")] )
self.declExpr3b = self.makeVariableDeclExpr( [("a", "1"), ("b","b")] )
+ self.declExpr4 = self.makeVariableDeclExpr( [("vector", "x")] )
self.identifiers = {}
! for name in [ "var1", "var2", "var3", "x", "y", "z", "a", "b", "c", "vector", "string" ]:
for count in range(1,10):
variable_name = name + "." + str(count)
***************
*** 174,180 ****
pass
! def declareVariables( self, names, declExpr ):
for name in names:
! self.resolver.declareVariable( name, declExpr )
def resolveVariables( self, variables ):
--- 176,188 ----
pass
! def declareVariables( self, names, decl_expr ):
for name in names:
! found = False
! for declarator in decl_expr.getDeclarators():
! if declarator.name.identifier == name:
! self.resolver.declareVariable( declarator.name, decl_expr )
! self.identifiers[ name + ".0" ] = declarator.name
! found = True
! self.assertEqual( found, True, "variable '%s' not found in the specified declaration expression" % name )
def resolveVariables( self, variables ):
***************
*** 183,193 ****
self.resolver.resolveVariable( variable[0], self.identifiers[id] )
! def checkVariableResolutions( self, names, declExpr ):
for name in names:
self.assertEqual( self.resolver.findVariable( self.identifiers[name] ),
! (IDTYPE_LOCALVARIABLE,declExpr) )
def testConstruction( self ):
self.assertEqual( self.resolver.current_scope.function, self.function )
def testResolveWithNoSubScope( self ):
--- 191,229 ----
self.resolver.resolveVariable( variable[0], self.identifiers[id] )
! def checkVariableDeclResolutions( self, names, decl_expr ):
! for name in names:
! declarator = self.findDeclarator( name, decl_expr )
! self.assertEqual( self.resolver.findVariable( declarator.name ),
! (IDTYPE_LOCALVARIABLE,declarator.name ) )
!
! def findDeclarator( self, name, decl_expr ):
! for declarator in decl_expr.getDeclarators():
! if declarator.name.identifier == name:
! return declarator
! self.fail( 'no declarator found for ' + name )
!
! def checkVariableResolutions( self, names, decl_expr ):
for name in names:
self.assertEqual( self.resolver.findVariable( self.identifiers[name] ),
! (IDTYPE_LOCALVARIABLE,decl_expr) )
!
! def checkUsingDeclResolution( self, name, using_expr ):
! self.assertEqual( self.resolver.findVariable( self.identifiers[name] ),
! (IDTYPE_IMPORTEDID,using_expr) )
!
! def checkAllScopeExited( self ):
! self.assertEqual( self.resolver.current_scope, self.function_scope )
def testConstruction( self ):
self.assertEqual( self.resolver.current_scope.function, self.function )
+ self.checkAllScopeExited()
+
+ def testResolveVariableDecl( self ):
+ self.resolver.enterScope()
+ self.declareVariables( ["var1"], self.declExpr1 )
+ self.resolver.exitScope()
+ self.checkAllScopeExited()
+
+ self.checkVariableDeclResolutions( ["var1"], self.declExpr1 )
def testResolveWithNoSubScope( self ):
***************
*** 195,203 ****
self.declareVariables( ["var1", "var2", "var3"], self.declExpr1 )
self.declareVariables( ["x", "y", "z"], self.declExpr2 )
! self.resolveVariables( [ ("var1",1), ("var2",1), ("var3",1), ("var2",2), ("var3",2), ("var3",3), ("var1",2) ] )
self.resolveVariables( [ ("x",1), ("y",1), ("x",2), ("z",1), ("z",2), ("y",1) ] )
self.resolver.exitScope()
! self.checkVariableResolutions( ["var1.1", "var2.1", "var1.2", "var2.2", "var3.1", "var3.2", "var3.3"], self.declExpr1 )
self.checkVariableResolutions( ["x.1", "x.2", "y.1", "z.1", "z.2"], self.declExpr2 )
--- 231,242 ----
self.declareVariables( ["var1", "var2", "var3"], self.declExpr1 )
self.declareVariables( ["x", "y", "z"], self.declExpr2 )
! self.resolveVariables( [ ("var1",1), ("var2",1), ("var3",1), ("var2",2),
! ("var3",2), ("var3",3), ("var1",2) ] )
self.resolveVariables( [ ("x",1), ("y",1), ("x",2), ("z",1), ("z",2), ("y",1) ] )
self.resolver.exitScope()
+ self.checkAllScopeExited()
! self.checkVariableResolutions( ["var1.1", "var2.1", "var1.2", "var2.2",
! "var3.1", "var3.2", "var3.3"], self.declExpr1 )
self.checkVariableResolutions( ["x.1", "x.2", "y.1", "z.1", "z.2"], self.declExpr2 )
***************
*** 215,223 ****
self.resolveVariables( [ ("var1",5), ("var2",5) ] )
self.resolver.exitScope()
self.checkVariableResolutions( ["var1.1", "var1.2", "var1.5",
"var2.1", "var2.2", "var2.3", "var2.4", "var2.5"], self.declExpr1 )
self.checkVariableResolutions( ["var1.3", "var1.4"], self.declExpr1b )
!
def suite():
--- 254,298 ----
self.resolveVariables( [ ("var1",5), ("var2",5) ] )
self.resolver.exitScope()
+ self.checkAllScopeExited()
self.checkVariableResolutions( ["var1.1", "var1.2", "var1.5",
"var2.1", "var2.2", "var2.3", "var2.4", "var2.5"], self.declExpr1 )
self.checkVariableResolutions( ["var1.3", "var1.4"], self.declExpr1b )
!
! def testResolveUsing( self ):
! self.resolver.enterScope()
! using_decl_expr = self.makeTrue() # dummy place holder
! self.resolver.usingDeclaration( IdentifierRef('std::vector'), using_decl_expr )
! self.resolveVariables( [ ("vector",1) ] )
! self.resolver.exitScope()
! self.checkAllScopeExited()
!
! self.checkUsingDeclResolution( "vector.1", using_decl_expr )
!
! def testResolveUsingGoesOutOfScope( self ):
! self.resolver.enterScope()
! self.declareVariables( ["vector"], self.declExpr4 )
! self.resolveVariables( [ ("vector",1) ] )
! self.resolver.enterScope()
! using_decl_expr = self.makeTrue() # dummy place holder
! self.resolver.usingDeclaration( IdentifierRef('std::vector'), using_decl_expr )
! self.resolveVariables( [ ("vector",2) ] )
! self.resolver.exitScope()
! self.resolveVariables( [ ("vector",3) ] )
! self.resolver.exitScope()
! self.checkAllScopeExited()
!
! self.checkUsingDeclResolution( "vector.2", using_decl_expr )
! self.checkVariableResolutions( ["vector.1", "vector.3"], self.declExpr4 )
!
! class IdentifierTest(unittest.TestCase):
! """IdentifierRef tests"""
!
! def testName( self ):
! self.assertEqual( IdentifierRef( 'std::vector' ).name(), 'vector' )
! self.assertEqual( IdentifierRef( 'std::tr1::vector' ).name(), 'vector' )
! self.assertEqual( IdentifierRef( '::tr1::vector' ).name(), 'vector' )
! self.assertEqual( IdentifierRef( '::vector' ).name(), 'vector' )
! self.assertEqual( IdentifierRef( 'vector' ).name(), 'vector' )
def suite():
|
|
From: <bl...@us...> - 2003-07-31 08:00:09
|
Update of /cvsroot/cpptool/rfta/src/pyrfta/test/rfta
In directory sc8-pr-cvs1:/tmp/cvs-serv22556
Added Files:
analyser.py node.py parser.py parsertest.py project.py
refactoring.py
Log Message:
* added a rough squetch of an attempt to make a generic interface to refactoring operation (to simplify language binding)
* added a port of the MiniParser framework with a tokenizer and node transform system
--- NEW FILE: analyser.py ---
class Analyser:
def __init__( self, document ):
class Symbol:
def __init__( self ):
pass
class LocaleVariableDeclaratorExpr:
def __init__( self ):
self.declarators = []
def getBaseType( self ):
pass
def getDeclarators( self ):
return self.declarators
class LocaleVariableDeclarator:
def getDeclarator( self ):
pass
def getBaseTypeSuffix( self ):
pass
def getArrayType( self ):
pass
# composed type: base + prefix + suffix
def getType( self ):
pass
def getName( self ):
pass
def getInitializer( self ):
pass
class LocaleVariableRef(Symbol):
def __init__( self, node ):
self.node = node
self.occurrences = [ node ]
def getName( self ):
return node.getName()
def setName( self ):
pass
# composed type: base + prefix + suffix
def getType( self ):
pass
def getDeclarator( self ):
pass
def getBaseTypeSuffix( self ):
pass
def getArrayType( self ):
pass
def getOccurrences( self ):
return self.occurrences
class FunctionBody(Symbol):
def __init__( self, node ):
self.node = node
def getName( self ):
pass
def setName( self ):
pass
def getModifiers( self ):
pass
def getParameters( self ):
pass
def getBody( self ):
pass
class MemberFunctionBody(FunctionBody):
def __init__( self, node ):
self.node = node
def getClass( self ):
pass
def getMemberInitializers( self ):
pass
def
--- NEW FILE: node.py ---
# node stuff
class Range:
def __init__( self, start_index, end_index ):
self.start_index = start_index
self.end_index = end_index
def getStartIndex( self ):
return self.start_index
def setStartIndex( self, start_index ):
self.start_index = start_index
def getEndIndex( self ):
return self.end_index
def setEndIndex( self, end_index ):
self.end_index = end_index
def getLength( self ):
return self.end_index - self.start_index
def setLength( self, length ):
self.end_index = self.start_index + length
def getTextRange( self, text ):
return text[ self.start_index : self.end_index ]
def isEmpty( self ):
return getLength() <= 0
def contains( self, other_range ):
if self.isEmpty() or other_range.isEmpty() or \
self.end_index ==other_range.start_index:
return False
return self.start_index >= other_range.start_index and \
self.end_index <= other_range.end_index
def overlap( self, other_range ):
return other_range.start_index < self.end_index and \
other_range.end_index > self.start_index:
def __str__( self ):
return "Range(%d,%d)" % (self.start_index,self.end_index)
class Node:
def __init__( self, manager ):
self.manager = manager
self.source = None
self.source_range = None
self.parent = None
def isFromSource( self ):
return self.source_range != None
def getSourceText( self ):
if self.source_range:
return self.source_range.getTextRange( self.source )
return ""
def getSourceRange( self ):
return self.source_range
def setSource( self, source, range ):
self.source = source
self.source_range = range
def accept( self, visitor ):
pass
def getParent( self ):
return self.parent
def getManager( self ):
return self.manager
def getEventManager( self ):
return self.manager.getNodeEventManager()
class CompositeNode(Node):
def __init__( self, manager ):
Node.__init__( self )
self.children = []
self.keyed_children = {}
def getChildren( self ):
return self.children
def getChildCount( self ):
return len( self.children )
def setChild( self, child, index ):
self.removeChildAt( index )
self.insertChildAt( child, index )
def getChildIndex( self, child ):
return self.children.index( child )
def insertChildAt( self, child, index ):
self.children[ index:index ] = [child]
child.parent = self
self.getEventManager().childInsertedAt( self, child, index )
def insertChildAfter( self, insert_after, child_to_insert ):
self.insertChild( getChildIndex(insert_after) + 1, child_to_insert )
def insertChildBefore( self, insert_before, child_to_insert ):
self.insertChild( getChildIndex(insert_before), child_to_insert )
def appendChild( self, child ):
self.insertChild( child, len(self.children) )
def removeChildAt( self, index ):
old_child = self.children[index]
self.children[index:index] = []
old_child.parent = None
self.getEventManager().childRemovedAt( self, old_child, index )
def removeChild( self, child ):
child_index = self.children.index( child )
self.removeChildAt( self, child_index )
def getKeyedChild( self, key ):
return self.keyed_children[ key ]
def getOptionalKeyedChild( self, key ):
return self.keyed_children.get( key, None )
def setKeyedChild( self, key, child ):
old_child = self.keyed_children.get( key, None )
if old_child:
self.keyed_children[ key ] = child
old_child.parent = None
self.getEventManager().childReplaced( self, key, old_child, child )
elif child:
self.keyed_children[ key ] = child
child.parent = self
self.getEventManager().childAdded( self, key, child )
def removeKeyedChild( self, key ):
if key in self.keyed_children:
self.keyed_children[ key ].parent = None
del self.keyed_children[ key ]
self.getEventManager().childRemoved( self, key, old_child )
def acceptChildren( self, visitor ):
for node in self.keyed_children.values():
node.accept( visitor )
for node in self.children:
node.accept( visitor )
# -------------------------
# Leaf Nodes
# -------------------------
class TextNodeMixIn:
def __init__( self, text = None ):
self.text = text
def getText( self ):
if self.isFromSource():
return self.getSourceText()
else:
return self.text
def setText( self, text ):
self.text = text
class Text(Node,TextNodeMixIn):
def __init__( self, manager, text = None ):
Node.__init__( self, manager )
TextNodeMixIn.__init__( self, text )
def accept( self, visitor ):
visitor.visitText( self )
class Identifier(Text):
def __init__( self, manager, identifier = None ):
Text.__init__( self, manager, identifier )
def accept( self, visitor ):
visitor.visitIdentifier( self )
def resolve( self ):
pass
# -------------------------
# Statements Nodes
# -------------------------
class BlockStatement(CompositeNode):
def __init__( self, manager ):
CompositeNode.__init__( self, manager )
def accept( self, visitor ):
visitor.visitBlockStatement( self )
class IfStatement(CompositeNode):
def __init__( self, manager, condition_node, then_node, else_node = None ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "condition", condition_node )
self.setKeyedChild( "then", then_node )
self.setKeyedChild( "else", else_node )
def getCondition( self ):
return self.getKeyedChild( "condition" )
def setCondition( self, condition_node ):
self.setKeyedChild( "condition", condition_node )
def getThen( self ):
return self.getKeyedChild( "then" )
def setThen( self, then_node ):
self.setKeyedChild( "then", then_node )
def getElse( self ):
return self.getOptionalKeyedChild( "else" )
def setElse( self, else_node ):
self.setKeyedChild( "else", else_node )
def accept( self, visitor ):
visitor.visitIfStatement( self )
class DeclarationStatement(CompositeNode):
def __init__( self, manager, declaration_node ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "declaration", declaration_node )
def getDeclaration( self ):
return self.getKeyedChild( "declaration" )
def setDeclaration( self, declaration_node ):
self.setKeyedChild( declaration_node )
def accept( self, visitor ):
visitor.visitDeclarationStatement( self )
class ExpressionStatement(CompositeNode):
def __init__( self, manager, expression ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "expression", expression )
def getExpression( self ):
return self.getKeyedChild( "expression" )
def setExpression( self, expression ):
self.setKeyedChild( "expression", expression )
def accept( self, visitor ):
visitor.visitExpressionStatement( self )
# --------------------------
# Expressions Nodes
# --------------------------
# generic partially parsed expression
# has identifiers and operators child nodes
class UnspecifiedExpression(CompositeNode,TextNodeMixIn):
def __init__( self, manager, text = None ):
CompositeNode.__init__( self, manager )
TextNodeMixIn.__init__( self, text )
def accept( self, visitor ):
visitor.visitUnspecifiedExpression( self )
class DeclarationExpression(CompositeNode):
def __init__( self, manager, base_type ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "base_type", base_type )
def getBaseType( self ):
return self.getKeyedChild( "base_type" )
def setBaseType( self, base_type ):
self.SetKeyedChild( "base_type", base_type )
def accept( self, visitor ):
visitor.visitDeclarationExpression( self )
class Declarator(CompositeNode):
def __init__( self, manager, name ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "name", name )
def getName( self ):
return self.getKeyedChild( "name" )
def getBaseTypeSuffix( self ):
return self.getKeyedChild( "base_type_suffix" )
def setBaseTypeSuffix( self, base_type_suffix ):
self.setKeyedChild( "base_type_suffix", base_type_suffix )
def getArrayType( self ):
return self.getOptionalKeyedChild( "array_type" )
def setArrayType( self, array_type ):
self.setKeyedChild( "array_type" )
def getInitializer( self ):
return self.getOptionalKeyedChild( "initializer" )
def setInitializer( self, initializer ):
self.setKeyedChild( "initializer" )
def accept( self, visitor ):
visitor.visitDeclarator( self )
class AssignInitializer(CompositeNode):
def __init__( self, manager, expression ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "expression", expression )
def getExpression( self ):
return self.getKeyedChild( "expresion" )
def setExpression( self, expression ):
self.setKeyedChild( "expression", expression )
def accept( self, visitor ):
visitor.visitAssignInitializer( self )
class ConstructInitializer(CompositeNode):
def __init__( self, manager, parameters ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "parameters", parameters )
def getParameters( self ):
return self.getKeyedChild( "parameters" )
def setParameters( self, parameters ):
self.setKeyedChild( "parameters", parameters )
def accept( self, visitor ):
visitor.visitConstructInitializer( self )
# ----------------------------
# Declarations
# ----------------------------
class Type(Text):
def __init__( self, manager, type = None ):
Text.__init__( self, manager, type )
def accept( self, visitor ):
visitor.visitType( self )
# static, virtual const, volatile, =0 modifiers.
class Modifier(Text):
def __init__( self, manager, modifier = None ):
Text.__init__( self, manager, modifier )
def accept( self, visitor ):
visitor.visitModifier( self )
class Parameter(CompositeNode):
def __init__( self, manager, type,
name = None,
array_type = None,
initializer = None ):
CompositeNode.__init__( self, manager )
self.setKeyedChild( "type", type )
self.setKeyedChild( "name", name )
self.setKeyedChild( "array_type", array_type )
self.setKeyedChild( "initializer", initializer )
def getType( self ):
return self.getKeyedChild( "type" )
def setType( self, type ):
self.setKeyedChild( "type", type )
def getName( self ):
return self.getOptionalKeyedChild( "name" )
def setName( self, name ):
self.setKeyedChild( "name", name ):
def getArrayType( self ):
return self.getOptionalKeyedChild( "array_type" )
def setArrayType( self, array_type ):
self.setKeyedChild( "array_type", array_type )
def getInitializer( self ):
return self.getOptionalKeyedChild( "initializer" )
def setInitializer( self, initializer ):
self.setKeyedChild( "initializer", initializer )
class Parameters(CompositeNode):
def __init__( self, manager ):
CompositeNode.__init__( self, manager )
class FunctionBodyDecl(CompositeNode):
def __init__( self, manager, body,
name, parameters,
return_type = None ):
CompositeNode.__init__( self, manager)
self.setKeyedChild( "name", name )
self.setKeyedChild( "parameters", parameters )
if not return_type:
return_type = Type( manager, "int" )
self.setKeyedChild( "return_type", return_type )
def getName( self ):
return self.getKeyedChild( "name" )
def setName( self, name ):
self.setKeyedChild( "name", name )
def getParameters( self ):
return self.getKeyedChidl( "parameters" )
def setParameters( self, parameters ):
self.setKeyedChild( "parameters", parameters" )
def getReturnType( self ):
return self.getReturnType( "return_type" )
def setReturnType( self, return_type ):
self.setKeyedChild( "return_type", return_type )
def getBody( self ):
return self.getKeyedChild( "body" )
def setBody( self, body ):
self.setKeyedChild( "body", body )
--- NEW FILE: parser.py ---
import re, string
cpp_keywords = """"__asm else main struct
__assume enum __multiple_inheritance switch
auto __except __single_inheritance template
__based explicit __virtual_inheritance this
bool extern mutable thread
break false naked throw
case __fastcall namespace true
catch __finally new try
__cdecl float noreturn __try
char for operator typedef
class friend private typeid
const goto protected typename
const_cast if public union
continue inline register unsigned
__declspec __inline reinterpret_cast using declaration,
using directive
default int return uuid
delete __int8 short __uuidof
dllexport __int16 signed virtual
dllimport __int32 sizeof void
do __int64 static volatile
double __leave static_cast wmain
dynamic_cast long __stdcall while"""
IDENTIFIER = "id"
STRING = "string"
CHAR = "char"
INTEGER = "integer"
FLOAT = "float"
SYMBOL = "symbol"
ERROR = "error"
CPP_COMMENT = "cpp_comment"
C_COMMENT = "c_comment"
PP_DIRECTIVE = "pp_directive"
SPACES = "spaces"
END_STREAM = "<<end>>"
class Token:
def __init__( self, type, start_pos, end_pos, source ):
self.type = type
self.start_pos = start_pos
self.end_pos = end_pos
self.source = source
def getType( self ):
return self.type
def getText( self ):
return self.source[self.start_pos:self.end_pos]
def getStartPos( self ):
return self.start_pos
def getEndPos( self ):
return self.end_pos
def getSource( self ):
return self.source
def __eq__( self, other ):
return self.type == other.type and self.start_pos == other.start_pos \
and self.end_pos == other.end_pos and self.source == other.source
def __ne__( self, other ):
return not self.__eq__( other )
def __repr__( self ):
return str(self)
def __str__( self ):
return "Token(%s, %d-%d = '%s')" % (self.type,self.start_pos,self.end_pos,self.getText())
class CppScanner:
def __init__( self ):
patterns = [ ( SPACES, '[\\s]+' ),
( IDENTIFIER, '[A-Za-z_][A-Za-z0-9_]*' ),
( STRING, '"' ),
( CHAR, "'" ),
( INTEGER, '[0-9]+L?' ),
( INTEGER, '0x[0-9A-Fa-f]+L?' ),
( FLOAT, '([0-9]+)?\\.[0-9]+(e[-+]?[0-9]+)?[fFL]?' ),
( FLOAT, '[0-9]+e[-+]?[0-9]+[fFL]?' ),
( SYMBOL, self._makeChoice( ':: << >> <= >= \\|\\| && &= \\|= \\+= -= /= \\*= \\^=' ) ),
( SYMBOL, self._makeChoice( '\\( \\) \\[ \\] \\{ \\} ~ / - \\+ / \\* = \\^ % < > & \\| ; , : \\?' ) ),
( CPP_COMMENT, '//' ),
( C_COMMENT, '/\\*' ),
( PP_DIRECTIVE, '#' ) ]
self.patterns = []
for pattern in patterns:
# print "compiling pattern: ", pattern[0], ':', pattern[1]
self.patterns.append( ( pattern[0], re.compile(pattern[1]) ) )
self.keywords = {}
for keyword in string.split(cpp_keywords):
self.keywords[keyword] = None
def _makeChoice( self, choices ):
return string.join( string.split( choices ), '|' )
def tokenize( self, text ):
self.pos = 0
tokens = []
last_token_is_error = False
while self.pos < len(text):
token = self._readOneToken( text )
if not token: # skipped
continue
if token.getType() == ERROR: # collate error token
if last_token_is_error:
tokens[-1] = Token( ERROR, tokens[-1].getStartPos(), token.getEndPos(), token.getSource() )
continue
last_token_is_error = True
else:
last_token_is_error = False
tokens.append( token )
return tokens
def _readOneToken( self, text ):
best_match = (0, None) # matched_len, matched_pattern
# print "Trying to match: '%s'" % text
for pattern in self.patterns:
match = pattern[1].match( text, self.pos )
if match:
# print "Match found: %s = %s %d-%d" % (pattern[0],text[match.start():match.end()],match.start(),match.end())
matched_len = match.end() - match.start()
if not best_match or best_match[0] < matched_len:
best_match = (matched_len,pattern)
token = None
if best_match[1]:
# print "Best match is: %s = %s" % (best_match[1],text[self.pos:self.pos + best_match[0]])
token = self._makeToken( best_match[1], self.pos, self.pos + best_match[0], text )
else:
# print "No match found => Error"
token = self._makeErrorToken( self.pos, self.pos+1, text )
if token:
self.pos = token.getEndPos()
# print "Generated token:", token
return token
def _makeToken( self, pattern, start_pos, end_pos, text ):
type = pattern[0]
if type == SPACES:
self.pos = end_pos
elif type == STRING:
return self._readStringToken( type, end_pos, text, '"' )
elif type == CHAR:
return self._readStringToken( type, end_pos, text, "'" )
elif type == CPP_COMMENT:
self._skipComment( end_pos, text, '\n' )
elif type == C_COMMENT:
self._skipComment( end_pos, text, '*/' )
elif type == PP_DIRECTIVE:
return self._readPreprocessorDirective( end_pos, text )
else:
if type == IDENTIFIER and text[start_pos:end_pos] in self.keywords:
type = SYMBOL
return Token( type, start_pos, end_pos, text )
return None
def _skipComment( self, pos, text, end_pattern ):
end_pos = string.find( text, end_pattern, pos )
if end_pos == -1:
self.pos = len(text)
else:
self.pos = end_pos + len(end_pattern)
def _readStringToken( self, type, pos, text, delimiter_char ):
# print "Trying to match string '%s' : %d" % (text,pos)
not_found = True
while not_found:
end_pos = string.find( text, delimiter_char, pos )
if end_pos == -1:
return self._makeErrorToken( self.pos, len(text), text )
if text[end_pos-1] != '\\':
not_found = False
pos = end_pos + 1
return Token( type, self.pos, pos, text )
def _readPreprocessorDirective( self, pos, text ):
start_pos = pos - 1
while True:
next_eol_pos = string.find( text, '\n', pos )
if next_eol_pos != -1:
next_comment_pos = string.find( text, '/*', pos, next_eol_pos )
if next_comment_pos != -1: # skip C comment
self._skipComment( next_comment_pos + 2, text, '*/' )
pos = self.pos
continue
if text[next_eol_pos-1] == '\\': # directive continue on next line
pos = next_eol_pos + 1
continue
pos = next_eol_pos + 1
else:
pos = len(text)
break
self.pos = pos
return Token( PP_DIRECTIVE, start_pos, pos, text )
def _makeErrorToken( self, start_pos, end_pos, text ):
return Token( ERROR, start_pos, end_pos, text )
class CppParser:
def __init__( self ):
pass
def parse( self, text ):
scanner = CppScanner()
self.tokens = scanner.tokenize()
class NodeBase:
def getType( self ):
pass
def getChildren( self ):
pass
def isLeaf( self ):
return False
def __repr__( self ):
return str(self)
def prettyPrint( self, margin = None, indentLevel = None ):
if margin == None:
margin = 0
if indentLevel == None:
indentLevel = 0
indentBase = " " * margin
indent = indentBase
if indentLevel > 0:
indent += "| " * (indentLevel-1) + "|-"
print indent + self._getPrettyNodeHeader()
indentLevel += 1
for child in self.getChildren():
if child:
child.prettyPrint( margin, indentLevel )
else:
print indent + "|-NONE"
class Node(NodeBase):
def __init__( self, type, children = None ):
self.type = type
if children:
self.children = children
else:
self.children = []
def getType( self ):
return self.type
def setType( self, type ):
self.type = type
def getChildren( self ):
return self.children
def addChild( self, child ):
self.children.append( child )
def _getPrettyNodeHeader( self ):
return "Node(%s : %d children)" % (self.type,len(self.children))
def __str__( self ):
return "Node(%s, %s)" % (self.type, str(self.children))
class TokenNode(NodeBase):
def __init__( self, token ):
self.token = token
def getType( self ):
return self.token.getType()
def getChildren( self ):
return []
def isLeaf( self ):
return True
def _getPrettyNodeHeader( self ):
return str(self)
def __str__( self ):
return "Token(%s,'%s')" % (self.getType(),self.token.getText())
class Match:
def __init__( self, success, value = None ):
self.success = success
self.value = value
def __nonzero__( self ):
return self.success
def getValue( self ):
return self.value
class TokenProvider:
def __init__( self, tokens ):
self.tokens = tokens
self.index = 0
self.states = []
self.end_stream_token = Token( END_STREAM, 0, 2+3+2, "<<end>>" )
def hasMoreToken( self ):
return self.index < len(self.tokens)
def getNextToken( self ):
if self.hasMoreToken():
token = self.tokens[ self.index ]
self.index += 1
else:
token = self.end_stream_token
return token
def pushState( self ):
self.states.append( self.index )
def popState( self ):
self.index = self.states.pop()
def commitState( self ):
self.states.pop()
class Parser:
""" Top-level class for all sub-parser.
This implements a recursive descendent parser with infinite look-ahead ( LL(n) parser ).
Notes that the Parser also overload the operator |, >> and []. They are not defined here for technical reason.
'parser1 | parser2' returns an AlternativeParser( parser1, parser2). If parser1 is already an AlternativeParser,
then parser2 is added to the list of alternative of parser1.
'parser1 >> parser2' returns an SequenceParser( parser1, parser2). If parser1 is already an SequenceParser,
then parser2 is added to the list of sub-parser of parser1.
'parser[action]' returns an ActionParser( parser, action ). action should be a callable which accepts the node
returned by parser as parameter. if action is a string, then a NameAction(action) is automatically created
to set the name of the node returned by parser.
"""
def __init__( self ):
self.action_on_match = None # not use, use a decoarator parser instead (ActionParser)
def parse( self, scanner ):
""" Called to attempt to parse the next tokens provided by scanner.
scanner should be a TokenProvider.
Returns a Match to indicates success or error, and on success, store the 'node tree' representing
the match the 'value' of Match.
On match failure, the parser is responsible to restore the scanner to its previous position
(see TokenProvider.pushState() & TokenProvider.popState()).
The method matched() & notMatched() should be called to created the Match object.
"""
pass
def matched( self, scanner, result_value ):
""" Called on parse success to create the Match object to return.
result_value is the node representing the matched parse tree.
"""
if self.action_on_match:
result_value = self.action_on_match( result_value )
return Match( True, result_value )
def notMatched( self ):
""" Called on parse failure to create the Match object."""
return Match( False )
class AlternativeParser(Parser):
def __init__( self, alt1, alt2 ):
Parser.__init__( self )
self.alternatives = [ alt1, alt2 ]
def parse( self, scanner ):
for alternative in self.alternatives:
scanner.pushState()
result = alternative.parse( scanner )
if result:
scanner.commitState()
return self.matched( scanner, result.getValue() )
scanner.popState()
return self.notMatched()
def addAlternative( self, alt ):
self.alternatives.append( alt )
return self
def __str__( self ):
stringized = []
for parser in self.alternatives:
stringized.append( str(parser) )
return "( %s )" % string.join( stringized, " | " )
class SequenceParser(Parser):
def __init__( self, parser1, parser2 ):
Parser.__init__( self )
self.subparsers = [ parser1, parser2 ]
self.node_name = '?'
def parse( self, scanner ):
results = Node( self.node_name )
for subparser in self.subparsers:
result = subparser.parse( scanner )
if not result:
return self.notMatched()
results.addChild( result.getValue() )
return self.matched( scanner, results )
def addSubParser( self, subparser ):
self.subparsers.append( subparser )
return self
def __str__( self ):
stringized = []
for parser in self.subparsers:
stringized.append( str(parser) )
return "( %s )" % string.join( stringized, " >> " )
class SymbolParser(Parser):
def __init__( self, text ):
Parser.__init__( self )
self.text = text
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != SYMBOL or token.getText() != self.text:
return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
def __str__( self ):
return '"%s"' % self.text
class ActionParser(Parser):
def __init__( self, parser, action ):
Parser.__init__( self )
self.parser = parser
self.action = action
def parse( self, scanner ):
result = self.parser.parse( scanner )
if result:
return self.matched( scanner, self.action( result.getValue() ) )
return self.notMatched()
def __str__( self ):
return "(%s)[action]" % str(self.parser)
class NameAction:
def __init__( self, name ):
self.name = name
def __call__( self, node ):
node.setType( self.name )
return node
def name_a( name ):
return NameAction( name )
def make_p( parser ):
if type(parser) == type(""):
return SymbolParser( parser )
return parser
# defined there so that Alternative and Sequence class have already been defined
def Parser__or__( self, other ):
other = make_p( other )
if isinstance( self, AlternativeParser ):
return self.addAlternative( other )
else:
return AlternativeParser( self, other )
def Parser__rshift__( self, other ):
other = make_p( other )
if isinstance( self, SequenceParser ):
return self.addSubParser( other )
else:
return SequenceParser( self, other )
def Parser__getitem__( self, action ):
if type(action) == type(""):
action = NameAction( action )
return ActionParser( self, action )
def injectParserOperatorMethods( parser_class ):
parser_class.__or__ = Parser__or__
parser_class.__rshift__ = Parser__rshift__
parser_class.__getitem__ = Parser__getitem__
for class_name in [ Parser, AlternativeParser, SequenceParser, SymbolParser, ActionParser ]:
injectParserOperatorMethods( class_name )
class OptionalParser(Parser):
def __init__( self, parser ):
Parser.__init__( self )
self.optional_parser = parser
def parse( self, scanner ):
scanner.pushState()
result = self.optional_parser.parse( scanner )
if result:
scanner.commitState()
return self.matched( scanner, result.getValue() )
scanner.popState()
return self.matched( scanner, None )
def __str__( self ):
return '%s?' % str(self.optional_parser)
class RepeatParser(Parser):
def __init__( self, min_count, parser ):
Parser.__init__( self )
self.min_count = min_count
self.parser = parser
def parse( self, scanner ):
results = Node( '?' )
repeat_count = 0
while True:
scanner.pushState()
result = self.parser.parse( scanner )
if not result:
scanner.popState()
break
results.addChild( result.getValue() )
repeat_count += 1
if repeat_count >= self.min_count:
return self.matched( scanner, results )
return self.notMatched()
def __str__( self ):
if self.min_count == 0:
return '%s*' % self.parser
elif min_count == 1:
return '%s+' % self.parser
return 'repeat(%d,%s)' % (self.min_count,str(self.parser))
class AnyUntil(Parser):
def __init__( self, until_parser ):
Parser.__init__( self )
self.until_parser = until_parser
def parse( self, scanner ):
results = Node( '?' )
while True:
scanner.pushState()
result = self.until_parser.parse( scanner )
scanner.popState()
if result:
break
results.addChild( TokenNode( scanner.getNextToken() ) )
return self.matched( scanner, results )
def __str__( self ):
return 'anyuntil( %s )' % str(self.until_parser)
class TerminalParser(Parser):
def __init__( self, type ):
Parser.__init__( self )
self.type = type
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != self.type:
return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
def __str__( self ):
return 'TOKEN(%s)' % str(self.type)
class ZeroParser(Parser):
def __init__( self ):
Parser.__init__( self )
def parse( self, scanner ):
token = scanner.getNextToken()
if token.getType() != INTEGER or token.getText() != '0':
return self.notMatched()
return self.matched( scanner, TokenNode( token ) )
def __str__( self ):
return '0' % str(self.type)
class InlineChildAction:
def __init__( self, node_index ):
self.node_index = node_index
def __call__( self, node ):
inlined_node = node.getChildren()[ self.node_index ]
node.getChildren()[self.node_index : self.node_index+1] = inlined_node.getChildren()
return node
class InlineChildrenAction:
def __call__( self, node ):
new_children = []
for child in node.getChildren():
if child.isLeaf():
new_children.append( child )
else:
new_children.extend( child.getChildren() )
new_node = Node(node.getType(), new_children)
return new_node
def inline_child_at_a( node_index ):
return InlineChildAction( node_index )
def inline_children_a():
"""Takes all the children of the chil nodes and bring them up by one level.
Root
|-ChildNode1
| |+ChildNode1.1
| |+ChildNode1.2
|-ChildNode2
| |+ChildNode2.1
|-LeafNode3
=>
Root
|+ChildNode1.1
|+ChildNode1.2
|+ChildNode2.1
|-LeafNode3
"""
return InlineChildrenAction()
end_p = TerminalParser( END_STREAM )
def symbol_p( symbol ):
""" Matchs the specified symbol.
symbol should be a string. """
return SymbolParser( symbol )
def maybe_p( parser ):
""" Matchs an optional parser.
parser should be a parser. """
return OptionalParser( make_p( parser ) )
def repeat_p( min_count, parser ):
""" Matchs at leat min_count occurrences of parser."""
return RepeatParser( min_count, parser )[inline_children_a()]
def list_p( repeated_parser, separator_parser ):
""" Matchs a 'list' of 1 or more occurrences of repeated_parser separated by separator_parser.
Examples: list_p( id_p, ',' ) will match 'v1,v2,v3'"""
return (make_p(repeated_parser) >> \
repeat_p( 0, make_p(separator_parser) >> \
make_p(repeated_parser) ) )[ inline_child_at_a(1) ]
def anyuntil_p( parser ):
""" Matchs any tokens until parser returns a match."""
return AnyUntil( make_p( parser ) )
id_p = TerminalParser( IDENTIFIER )
eos_p = symbol_p( ';' )
nested_id_p = (maybe_p( "::" ) >> list_p( id_p, symbol_p('::') ))[inline_child_at_a(1)]['nested_id']
# use separate parser for inheritance => allow other alternative matching
#template_instantiation_p = symbol_p("<") >> maybe_p( list_p( any_until_p( ","
#inherited_parent_class_p = nested_id_p >> maybe_p(template_instantiation_p)
null_statement_p = eos_p['null_statement']
basic_type_spec_p = symbol_p('char') | 'wchar_t' | 'bool' | 'short' | 'int' | 'long' \
| 'signed' | 'unsigned' | 'float' | 'double' | 'void'
type_p = basic_type_spec_p
type_suffix_p = repeat_p(0, symbol_p('*'))
array_type_p = repeat_p(0, symbol_p('[') >> anyuntil_p( symbol_p(']') ) >> symbol_p(']'))
#array_type_p = repeat_p(0, symbol_p('[') )
optional_initializer_p = maybe_p( symbol_p('=') >> anyuntil_p( symbol_p(',') | eos_p ) )
attribut_modifiers_p = repeat_p( 0, symbol_p('static') | 'mutable' | 'const' | 'volatile' )
#attribut_decl_p = attribut_modifiers_p >> type_p >> id_p >> array_type_p >> optional_initializer_p \
attribut_decl_p = attribut_modifiers_p >> type_p >> id_p >> array_type_p >> optional_initializer_p \
>> repeat_p( 0, symbol_p(',') >> type_suffix_p >> id_p >> array_type_p >> optional_initializer_p ) \
>> eos_p
function_parameters_decl = symbol_p( '(' ) >> ')' # need to consume everything balancing brace
cv_modifier_p = symbol_p('const') | 'volatile'
member_initializers_p = symbol_p(':') >> anyuntil_p('{')
statements_p = null_statement_p
block_statement_p = symbol_p( '{' ) >> statements_p >> '}'
function_body_p = block_statement_p
member_function_modifiers_p = symbol_p('static') | 'virtual'
function_decl_p = member_function_modifiers_p >> type_p >> id_p >> function_parameters_decl \
>> maybe_p( cv_modifier_p ) >> maybe_p( symbol_p('=') >> ZeroParser() ) \
>> ( ( maybe_p(member_initializers_p) >> function_body_p ) \
| eos_p )
friend_decl_p = symbol_p('friend') >> anyuntil_p( eos_p ) >> eos_p
access_level_spec_p = (symbol_p("public") | "protected" | "private") >> ':'
class_body_statements_p = null_statement_p \
| attribut_decl_p \
| access_level_spec_p \
| function_decl_p \
| friend_decl_p # enum, nested_class/struct, how ?
class_inheritance_and_body_decl = symbol_p( ":" ) >> anyuntil_p( "{" ) >> class_body_statements_p >> "}"
template_instantiation_param_p3 = anyuntil_p( symbol_p(">") )
template_instantiation_param_p2 = id_p >> "<" >> template_instantiation_param_p3 >> ">"
template_instantiation_param_p = anyuntil_p( symbol_p(",") | template_instantiation_param_p2 ) \
>> maybe_p(template_instantiation_param_p2)
class_specialization_p = symbol_p("<") >> list_p( template_instantiation_param_p, "," ) >> ">"
class_decl_p = (symbol_p("class") | "struct") >> nested_id_p >> maybe_p(class_specialization_p) \
>> maybe_p( class_inheritance_and_body_decl ) >> eos_p
statements = end_p \
| eos_p \
| TerminalParser( PP_DIRECTIVE ) \
| class_decl_p
--- NEW FILE: parsertest.py ---
import unittest
import mock
import parser
import string
class CppScannerTest(unittest.TestCase):
"""CppScanner tests"""
def setUp( self ):
self.scanner = parser.CppScanner()
def tearDown( self ):
self.scanner = None
def testSkipSpaces( self ):
self.checkEmptyParse( ' ' )
self.checkEmptyParse( ' ' )
self.checkEmptyParse( ' \t' )
self.checkEmptyParse( '\t \n' )
def testIdentifier( self ):
self.checkOneToken( 'a', parser.IDENTIFIER )
self.checkOneToken( 'A', parser.IDENTIFIER )
self.checkOneToken( '_', parser.IDENTIFIER )
self.checkOneToken( '_1', parser.IDENTIFIER )
self.checkOneToken( 'za', parser.IDENTIFIER )
self.checkOneToken( 'zA', parser.IDENTIFIER )
self.checkOneToken( 'c3', parser.IDENTIFIER )
self.checkOneToken( 'c3z', parser.IDENTIFIER )
self.checkOneToken( 'c3_', parser.IDENTIFIER )
def testString( self ):
self.checkOneToken( '""', parser.STRING )
self.checkOneToken( '"abc"', parser.STRING )
self.checkOneToken( '"\\""', parser.STRING )
self.checkOneToken( '"ab\\"cd\\""', parser.STRING )
def testChar( self ):
# allow for more than actual 'char' but we don't need to bother about that
self.checkOneToken( "''", parser.CHAR )
self.checkOneToken( "'a'", parser.CHAR )
self.checkOneToken( "'abc'", parser.CHAR )
self.checkOneToken( "'\\''", parser.CHAR )
def testInteger( self ):
self.checkOneToken( '0', parser.INTEGER )
self.checkOneToken( '123456789', parser.INTEGER )
self.checkOneToken( '1234L', parser.INTEGER )
self.checkOneToken( '0x1234L', parser.INTEGER )
def testFloat( self ):
self.checkOneToken( '0.0', parser.FLOAT )
self.checkOneToken( '0.0f', parser.FLOAT )
self.checkOneToken( '0.0F', parser.FLOAT )
self.checkOneToken( '0.0L', parser.FLOAT )
self.checkOneToken( '1344560.7894', parser.FLOAT )
self.checkOneToken( '1344560.7894e10', parser.FLOAT )
self.checkOneToken( '1344560.7894e-10', parser.FLOAT )
self.checkOneToken( '1344560.7894e+10', parser.FLOAT )
self.checkOneToken( '1344560e+10', parser.FLOAT )
self.checkOneToken( '.3', parser.FLOAT )
self.checkOneToken( '.3e10L', parser.FLOAT )
def testSymbol( self ):
symbols = string.split( ':: << >> <= >= || && &= |= += -= /= *= ^= ' \
'( ) [ ] { } ~ / - + / * = ^ % < > & | ; , : ? ' \
'if else break continue for while do void int' )
for symbol in symbols:
self.checkOneToken( symbol, parser.SYMBOL )
def testCppComment( self ):
self.checkEmptyParse( '//' )
self.checkEmptyParse( '////' )
self.checkEmptyParse( '////\n' )
def testCComment( self ):
self.checkEmptyParse( '/**/' )
self.checkEmptyParse( '/* //\n */' )
def testPreprocessorDirective( self ):
self.checkOneToken( '#include <vector>', parser.PP_DIRECTIVE )
self.checkOneToken( '#include <vector>\n', parser.PP_DIRECTIVE )
self.checkOneToken( '#define macro(x,y) \\n ((x) * (y))\n', parser.PP_DIRECTIVE )
self.checkOneToken( '#define macro(x,y) \\n /* ((x) * (y))\n // cpp comment\n */ x*y\n', parser.PP_DIRECTIVE )
def testTokenStream( self ):
self.checkManyTokens( 'namespace FS = boost::filesystem;', \
[ ('namespace',parser.SYMBOL,1), ('FS',parser.IDENTIFIER,1),
('=',parser.SYMBOL,1), ('boost',parser.IDENTIFIER,0),
('::',parser.SYMBOL,0), ('filesystem',parser.IDENTIFIER,0),
(';',parser.SYMBOL,0) ] )
def checkEmptyParse( self, text ):
tokens = self.scanner.tokenize( text )
self.assertEqual( [], tokens )
def checkOneToken( self, text, token_type ):
expected_tokens = [ parser.Token( token_type, 0, len(text), text ) ]
actual_tokens = self.scanner.tokenize( text )
self.assertEqual( expected_tokens, actual_tokens )
def checkManyTokens( self, text, tokens_def ):
expected_tokens = []
pos = 0
for token_def in tokens_def:
expected_tokens.append( parser.Token( token_def[1], pos, pos+len(token_def[0]), text ) )
pos += len(token_def[0]) + token_def[2]
actual_tokens = self.scanner.tokenize( text )
self.assertEqual( expected_tokens, actual_tokens )
# for index in range(0,len(expected_tokens)):
# self.assertEqual( expected_tokens[index], actual_tokens[index] )
class CppParserTest(unittest.TestCase):
"""CppParserTest tests"""
def setUp( self ):
self.scanner = parser.CppScanner()
def tearDown( self ):
self.scanner = None
def testParseBasicParsers( self ):
self.checkParser( "MyClass", parser.id_p )
self.checkParser( ",", parser.symbol_p(',') )
self.checkParser( "MyClass MyItem", parser.repeat_p( 0, parser.id_p ) )
self.checkParser( "v1,v2,v3", parser.list_p( parser.id_p, ',' ) )
self.checkParser( "int **myVar[7][12]", parser.type_p >> parser.type_suffix_p >> parser.id_p >> parser.array_type_p )
self.checkParser( "::NameSpace::MyClass::myVar", parser.nested_id_p )
def checkParser( self, text, tested_parser ):
tokens = self.scanner.tokenize( text )
scanner = parser.TokenProvider( tokens )
match = tested_parser.parse( scanner )
self.assert_( match, "nested_id_p should have matched source" )
remaining_tokens = []
while scanner.hasMoreToken() :
remaining_tokens.append( scanner.getNextToken() )
self.assertEqual( [], remaining_tokens )
if match.getValue():
print "Parsing '%s'" % text
match.getValue().prettyPrint( 4 )
return match.getValue()
def suite():
return unittest.makeSuite(CppScannerTest)
if __name__ == '__main__':
unittest.TextTestRunner().run(suite())
--- NEW FILE: project.py ---
import refactoring
class Project
def getActiveDocument( self ):
return None # to implement
def getRefactoring( self, refactoring_name ):
return refactoring.refactoring_classes[ refactoring_name ]( self )
--- NEW FILE: refactoring.py ---
# registry for refactoring classes (name => class)
refactoring_classes = {}
class Refactoring
def __init__( self, project ):
self.project = project
self.properties = {}
self.actions = {}
def setProperty( self, property_name, value ):
self.properties[ property_name ] = value
def getProperty( self, property_name ):
return self.properties[ property_name ]
def doAction( self, action_name ):
self.actions[ action_name ]()
def setAction( self, action_name, action_fn ):
self.actions[ action_name ] = action_fn
class RenameLocalVariableRefactoring(Refactoring):
def __init__( self, project ):
Refactoring.__init__( self, project )
setAction( "analyse", self.analyse )
setAction( "apply", self.apply )
# input properties:
# "document": document containing the variable to rename
# "location": index of the local variable (may be at the beginning or within the variable text
# output properties:
# "count": number of occurrences found
# "original_name": local variable name
def analyse( self ):
# self.ensurePropertySet( "location" )
self.analysis = self.project.getAnalysisInfo( self.project.getActiveDocument() )
self.function_body = self.analysis.getFunctionBodyContainning( self.getProperty( "location" ) )
#should that there is a function body
self.variable_info = self.analysis.getSymbolAt( self.getProperty( "location" ) )
usages = self.variable_info.getOccurrences()
new_name = self.getProperty( "new_name" )
for usage in usages:
usage.setName( new_name )
# input properties:
# "document": document containing the variable to rename
# "location": index of the local variable (may be at the beginning or within the variable text
# "new_name": new local variable name
# output properties:
def apply( self ):
pass
refactoring_classes["RenameLocalVariable"] = RenameLocalVariableRefactoring
|
