From: <fwi...@us...> - 2007-11-19 00:04:32
|
Revision: 3700 http://jython.svn.sourceforge.net/jython/?rev=3700&view=rev Author: fwierzbicki Date: 2007-11-18 16:04:31 -0800 (Sun, 18 Nov 2007) Log Message: ----------- getting pretty good walker recognition. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-18 22:20:45 UTC (rev 3699) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-19 00:04:31 UTC (rev 3700) @@ -169,7 +169,7 @@ | DOUBLESTAR kwargs=NAME )? )? - -> ^(Arguments defparameter*) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + -> ^(Arguments defparameter+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? | STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? | DOUBLESTAR kwargs=NAME @@ -190,6 +190,7 @@ stmt : simple_stmt -> ^(Stmt simple_stmt) | compound_stmt + -> ^(Stmt compound_stmt) ; simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? NEWLINE Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-18 22:20:45 UTC (rev 3699) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-19 00:04:31 UTC (rev 3700) @@ -10,7 +10,7 @@ } module - : (^(Stmt stmt))+ { + : stmt+ { //System.out.println("<STMT>"); } ; @@ -22,7 +22,7 @@ ; varargslist - : ^(Arguments defparameter) { + : ^(Arguments defparameter*) (^(StarArgs sname=NAME))? (^(KWArgs kname=NAME))? { } | ^(StarArgs sname=NAME) (^(KWArgs kname=NAME))? { } @@ -35,7 +35,11 @@ } ; -stmt +stmt: + ^(Stmt any_stmt+) + ; + +any_stmt : expr_stmt | print_stmt | del_stmt @@ -56,8 +60,6 @@ | ^(Expr ^(augassign targ=test value=test)) { } | ^(Expr ^(Assign targs=test value=test)) { - //System.out.println("<Assign>"); - System.out.println($targs.text + " = " + $value.text); } ; @@ -203,18 +205,11 @@ ; classdef - : ^(ClassDef ^(Name NAME) (^(Bases bases))? ^(Body suite)) { + : ^(ClassDef ^(Name NAME) (^(Bases testlist))? ^(Body suite)) { + //System.out.println("class "); } ; -bases - : base+ {} - ; - -base - : NAME {} - ; - arglist: argument (COMMA argument)* ( COMMA ( STAR test (COMMA DOUBLESTAR test)? This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-19 15:13:18
|
Revision: 3704 http://jython.svn.sourceforge.net/jython/?rev=3704&view=rev Author: fwierzbicki Date: 2007-11-19 07:13:17 -0800 (Mon, 19 Nov 2007) Log Message: ----------- some if/elif/else support Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-19 03:48:52 UTC (rev 3703) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-19 15:13:17 UTC (rev 3704) @@ -100,6 +100,7 @@ Dict; If; Else; + Elif; While; Pass; Print; @@ -305,7 +306,7 @@ -> ^(Assert $t1 $t2?) ; -compound_stmt : if_stmt -> ^(If if_stmt) +compound_stmt : if_stmt | while_stmt | for_stmt | try_stmt -> ^(Try try_stmt) @@ -313,9 +314,14 @@ | classdef ; -if_stmt: 'if' test COLON! s1=suite ('elif' test COLON! suite)* ('else' COLON! suite)? +if_stmt: 'if' test COLON ifsuite=suite elif_clause* ('else' COLON elsesuite=suite)? + -> ^(If test $ifsuite elif_clause* ^(Else $elsesuite)?) ; +elif_clause : 'elif' test COLON suite + -> ^(Elif test suite) + ; + while_stmt : 'while' test COLON s1=suite ('else' COLON s2=suite)? -> ^(While test $s1 ^(Else $s2)?) ; @@ -349,7 +355,7 @@ | comparison ; -comparison: expr (comp_op expr)* +comparison: expr (comp_op^ expr)* ; //comparison : expr Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-19 03:48:52 UTC (rev 3703) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-19 15:13:17 UTC (rev 3704) @@ -51,6 +51,7 @@ // | assert_stmt | funcdef | classdef + | if_stmt ; expr_stmt @@ -79,7 +80,7 @@ ; print_stmt - : ^(Print RIGHTSHIFT? testlist) + : ^(Print RIGHTSHIFT? testlist?) { } ; @@ -130,6 +131,12 @@ } ; +if_stmt: ^(If test suite elif_clause* (^(Else suite))?) + ; + +elif_clause : ^(Elif test suite) + ; + suite : INDENT stmt+ DEDENT ; @@ -139,8 +146,9 @@ : ^('and' test test) {} | ^('or' test test) {} | ^('not' test) {} - | ^(comp_op left=test targs=test) {} + | ^(comp_op left=test targs=test) | ^(PLUS test test) + | ^(MINUS left=test right=test) {System.out.println("MINUS left:" + $left.text + "; right:" + $right.text);} | atom (trailer)* {} ; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-20 01:55:22
|
Revision: 3705 http://jython.svn.sourceforge.net/jython/?rev=3705&view=rev Author: fwierzbicki Date: 2007-11-19 17:55:20 -0800 (Mon, 19 Nov 2007) Log Message: ----------- More walking rules -- starting to walk some of the python 2.3 lib files without error -- most errors look like they are coming from List Comps. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-19 15:13:17 UTC (rev 3704) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-20 01:55:20 UTC (rev 3705) @@ -78,6 +78,7 @@ DEDENT; Test; + Msg; Stmt; Import; ImportFrom; @@ -104,12 +105,15 @@ While; Pass; Print; - Try; + TryExcept; + TryFinally; + Except; For; Return; Yield; String; IsNot; + In; NotIn; Raise; Type; @@ -131,6 +135,12 @@ Start; End; SliceOp; + UnaryPlus; + UnaryMinus; + UnaryTilde; + Delete; + Default; + Parens; } @header { @@ -216,7 +226,7 @@ ; expr_stmt : lhs=testlist - ( (augassign testlist -> ^(augassign $lhs testlist)) + ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) | ((ASSIGN rhs=testlist)+ -> ^(Assign ^(Targets $lhs) ^(Values $rhs))) | -> $lhs ) @@ -244,6 +254,7 @@ ; del_stmt : 'del' exprlist + -> ^(Delete exprlist) ; pass_stmt : 'pass' @@ -303,13 +314,13 @@ ; assert_stmt : 'assert' t1=test (COMMA t2=test)? - -> ^(Assert $t1 $t2?) + -> ^(Assert ^(Test $t1) ^(Msg $t2)?) ; compound_stmt : if_stmt | while_stmt | for_stmt - | try_stmt -> ^(Try try_stmt) + | try_stmt | funcdef | classdef ; @@ -323,24 +334,26 @@ ; while_stmt : 'while' test COLON s1=suite ('else' COLON s2=suite)? - -> ^(While test $s1 ^(Else $s2)?) + -> ^(While test ^(Body $s1) ^(Else $s2)?) ; for_stmt : 'for' exprlist 'in' testlist COLON s1=suite ('else' COLON s2=suite)? - -> ^(For exprlist ^('in' testlist) ^(Body $s1) ^(Else $s2)?) + -> ^(For exprlist ^(In testlist) ^(Body $s1) ^(Else $s2)?) ; -try_stmt : 'try' COLON! suite - ( (except_clause COLON! suite)+ ('else' COLON! suite)? - | 'finally' COLON! suite +try_stmt : 'try' COLON trysuite=suite + ( (except_clause+ ('else' COLON elsesuite=suite)? + -> ^(TryExcept ^(Body $trysuite) except_clause+ ^(Else $elsesuite)?)) + | ('finally' COLON suite + -> ^(TryFinally suite)) ) ; -except_clause : 'except' (t1=test (COMMA t2=test)?)? - -> 'except' ^(Type $t1)? ^(Name $t2)? +except_clause : 'except' (t1=test (COMMA t2=test)?)? COLON suite + -> ^(Except ^(Type $t1)? ^(Name $t2)? ^(Body suite)) ; -suite : simple_stmt +suite : simple_stmt -> ^(Stmt simple_stmt) | NEWLINE! INDENT (stmt)+ DEDENT ; @@ -403,7 +416,9 @@ term : factor ((STAR^ | SLASH^ | PERCENT^ | DOUBLESLASH^ ) factor)* ; -factor : (PLUS^|MINUS^|TILDE^) factor +factor : PLUS factor -> ^(UnaryPlus factor) + | MINUS factor -> ^(UnaryMinus factor) + | TILDE factor -> ^(UnaryTilde factor) | power ; @@ -411,7 +426,7 @@ -> atom (trailer)* (DOUBLESTAR factor)? ; -atom : LPAREN (testlist)? RPAREN -> ^(Tuple testlist?) +atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) @@ -478,8 +493,8 @@ -> ^(KWArgs $kwargs) ; -argument : test (ASSIGN test)? - -> ^(Arg test+) +argument : t1=test (ASSIGN t2=test)? + -> ^(Arg $t1 ^(Default $t2)?) ; list_iter : list_for Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-19 15:13:17 UTC (rev 3704) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-20 01:55:20 UTC (rev 3705) @@ -10,7 +10,7 @@ } module - : stmt+ { + : stmt* { //System.out.println("<STMT>"); } ; @@ -39,24 +39,26 @@ ^(Stmt any_stmt+) ; -any_stmt +any_stmt //combines simple_stmt and compound_stmt from Python.g : expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt -// | global_stmt -// | exec_stmt -// | assert_stmt + | global_stmt + | exec_stmt + | assert_stmt | funcdef | classdef | if_stmt + | while_stmt + | for_stmt + | try_stmt ; expr_stmt : ^(Expr testlist) { - //System.out.println("<testlist>"); } | ^(Expr ^(augassign targ=test value=test)) { } @@ -85,7 +87,7 @@ } ; -del_stmt : 'del' testlist +del_stmt : ^(Delete exprlist) ; pass_stmt : Pass { @@ -95,7 +97,7 @@ flow_stmt : break_stmt | continue_stmt | return_stmt -// | raise_stmt + | raise_stmt | yield_stmt ; @@ -111,6 +113,9 @@ yield_stmt : 'yield' testlist ; +raise_stmt: ^(Raise (^(Type test))? (^(Inst test))? (^(Tback test))?) + ; + import_stmt : ^(Import dotted_as_name+) { } @@ -131,24 +136,68 @@ } ; +global_stmt : ^(Global NAME+) + ; + +exec_stmt : ^(Exec test (^(Globals test))? (^(Locals test))?) + ; + +assert_stmt : ^(Assert ^(Test test) (^(Msg test))?) + ; + + if_stmt: ^(If test suite elif_clause* (^(Else suite))?) ; elif_clause : ^(Elif test suite) ; + +while_stmt : ^(While test ^(Body suite) (^(Else suite))?) + ; + +for_stmt : ^(For exprlist ^(In testlist) ^(Body suite) (^(Else suite))?) + ; + +try_stmt : ^(TryExcept ^(Body suite) except_clause+ (^(Else suite))?) + | ^(TryFinally suite) + ; + +except_clause : ^(Except (^(Type test))? (^(Name test))? ^(Body suite)) + ; suite : INDENT stmt+ DEDENT + | stmt+ ; //FIXME: lots of placeholders test -: ^('and' test test) {} + : ^('and' test test) {} | ^('or' test test) {} | ^('not' test) {} | ^(comp_op left=test targs=test) | ^(PLUS test test) - | ^(MINUS left=test right=test) {System.out.println("MINUS left:" + $left.text + "; right:" + $right.text);} + | ^(MINUS left=test right=test) {} + | ^(AMPER test test) + | ^(VBAR test test) + | ^(LEFTSHIFT test test) + | ^(RIGHTSHIFT test test) + | lambdef + | term + ; + +term + : ^(STAR factor factor) + | ^(SLASH factor factor) + | ^(PERCENT factor factor) + | ^(DOUBLESLASH factor factor) + | factor + ; + +factor + : ^(UnaryPlus factor) + | ^(UnaryMinus factor) + | ^(UnaryTilde factor) | atom (trailer)* {} ; @@ -168,12 +217,10 @@ //FIXME: lots of placeholders atom - : ^(List testlist?) { - } - | ^(Tuple testlist?) { - } - | ^(Dict testlist?) { - } + : ^(List testlist?) {} + | ^(Parens testlist?) {} + | ^(Dict testlist?) {} + | ^(Repr testlist?) {} | NAME {} | INT {} | LONGINT {} @@ -191,6 +238,9 @@ : STRING {} ; +lambdef: ^(Lambda varargslist? ^(Body test)) + ; + trailer : ^(ArgList arglist?) | ^(SubscriptList subscriptlist) @@ -205,6 +255,9 @@ | ^(Subscript (^(Start test))? (^(End test))? (^(SliceOp test?))?) ; +exprlist : ^(ExprList test+) + ; + testlist : test+ {} ; @@ -221,14 +274,14 @@ | ^(KWArgs test) ; -argument : ^(Arg test (ASSIGN test)?) +argument : ^(Arg test (^(Default test))?) ; list_iter: list_for | list_if ; -list_for: 'for' testlist 'in' testlist (list_iter)? +list_for: 'for' exprlist 'in' testlist (list_iter)? ; list_if: 'if' test (list_iter)? This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-20 04:13:55
|
Revision: 3707 http://jython.svn.sourceforge.net/jython/?rev=3707&view=rev Author: fwierzbicki Date: 2007-11-19 20:13:54 -0800 (Mon, 19 Nov 2007) Log Message: ----------- walks more than half of the python 2.3 lib without error. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-20 03:54:39 UTC (rev 3706) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-20 04:13:54 UTC (rev 3707) @@ -438,7 +438,10 @@ | (STRING)+ -> ^(String STRING+) ; -listmaker : test ( list_for | (options {greedy=true;}:COMMA! test)* ) (COMMA!)? +listmaker : test + ( list_for -> ^(ListComp list_for) + | (options {greedy=true;}:COMMA test)* -> test+ + ) (COMMA)? ; lambdef: 'lambda' (varargslist)? COLON test @@ -498,9 +501,7 @@ ; list_iter : list_for - -> ^(ListComp list_for) | list_if - -> ^(ListComp list_if) ; list_for : 'for' exprlist 'in' testlist (list_iter)? Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-20 03:54:39 UTC (rev 3706) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-20 04:13:54 UTC (rev 3707) @@ -218,6 +218,7 @@ //FIXME: lots of placeholders atom : ^(List testlist?) {} + | ^(ListComp list_for) {} | ^(Parens testlist?) {} | ^(Dict testlist?) {} | ^(Repr testlist?) {} This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-21 02:54:39
|
Revision: 3710 http://jython.svn.sourceforge.net/jython/?rev=3710&view=rev Author: fwierzbicki Date: 2007-11-20 18:54:33 -0800 (Tue, 20 Nov 2007) Log Message: ----------- saved current work as Python23.g and PythonWalker23.g and now trying to add some 2.5 features. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Added Paths: ----------- trunk/sandbox/wierzbicki/antlr/grammar/Python23.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-20 23:59:51 UTC (rev 3709) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-21 02:54:33 UTC (rev 3710) @@ -141,6 +141,12 @@ Delete; Default; Parens; + Alias; + Asname; + Decorator; + Decorators; + With; + GenExpFor; } @header { @@ -161,25 +167,40 @@ int startPos=-1; } +//single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE single_input : NEWLINE! | simple_stmt | compound_stmt NEWLINE! ; +//file_input: (NEWLINE | stmt)* ENDMARKER file_input : (NEWLINE! | stmt)* ; +//eval_input: testlist NEWLINE* ENDMARKER eval_input : (NEWLINE!)* testlist (NEWLINE!)* ; -funcdef : 'def' NAME parameters COLON suite - -> ^(FunctionDef ^(Name NAME) ^(Args parameters) ^(Body suite)) +//decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorator: AT dotted_name (LPAREN arglist? RPAREN)? NEWLINE + -> ^(Decorator dotted_name ^(ArgList arglist?)) + ; + +//decorators: decorator+ +decorators: decorator+ + ; + +//funcdef: [decorators] 'def' NAME parameters ':' suite +funcdef : decorators? 'def' NAME parameters COLON suite + -> ^(FunctionDef ^(Decorators decorators)? ^(Name NAME) ^(Args parameters) ^(Body suite)) ; +//parameters: '(' [varargslist] ')' parameters : LPAREN (varargslist)? RPAREN -> (varargslist)? ; +//varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] varargslist : defparameter (options {greedy=true;}:COMMA defparameter)* (COMMA ( STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? @@ -193,27 +214,33 @@ -> ^(KWArgs $kwargs) ; +//not in CPython's Grammar file defparameter : fpdef (ASSIGN test)? ; +//fpdef: NAME | '(' fplist ')' fpdef : NAME | LPAREN! fplist RPAREN! ; +//fplist: fpdef (',' fpdef)* [','] fplist : fpdef (options {greedy=true;}:COMMA fpdef)* (COMMA)? -> fpdef+ ; +//stmt: simple_stmt | compound_stmt stmt : simple_stmt -> ^(Stmt simple_stmt) | compound_stmt -> ^(Stmt compound_stmt) ; +//simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? NEWLINE -> small_stmt+ ; +//small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt small_stmt : expr_stmt -> ^(Expr expr_stmt) | print_stmt | del_stmt @@ -225,13 +252,17 @@ | assert_stmt ; +//expr_stmt: testlist (augassign testlist | ('=' testlist)*) expr_stmt : lhs=testlist - ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + ( (augassign yield_expr -> ^(augassign $lhs yield_expr)) + | (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + | ((ASSIGN yield_expr)+ -> ^(Assign ^(Targets $lhs) ^(Values yield_expr))) | ((ASSIGN rhs=testlist)+ -> ^(Assign ^(Targets $lhs) ^(Values $rhs))) | -> $lhs ) ; +//augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' augassign : PLUSEQUAL | MINUSEQUAL | STAREQUAL @@ -246,6 +277,7 @@ | DOUBLESLASHEQUAL ; +//print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) print_stmt : 'print' ( testlist | RIGHTSHIFT testlist @@ -253,14 +285,17 @@ -> ^(Print RIGHTSHIFT? testlist?) ; +//del_stmt: 'del' exprlist del_stmt : 'del' exprlist -> ^(Delete exprlist) ; +//pass_stmt: 'pass' pass_stmt : 'pass' -> Pass ; +//flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt flow_stmt : break_stmt | continue_stmt | return_stmt @@ -268,79 +303,119 @@ | yield_stmt ; +//break_stmt: 'break' break_stmt : 'break' ; +//continue_stmt: 'continue' continue_stmt : 'continue' ; +//return_stmt: 'return' [testlist] return_stmt : 'return' (testlist)? -> ^(Return testlist?) ; -yield_stmt : 'yield' testlist - -> ^(Yield testlist) +//yield_stmt: yield_expr +yield_stmt : yield_expr ; +//raise_stmt: 'raise' [test [',' test [',' test]]] raise_stmt: 'raise' (t1=test (COMMA t2=test (COMMA t3=test)?)?)? -> ^(Raise ^(Type $t1)? ^(Inst $t2)? ^(Tback $t3)?) ; -import_stmt : 'import' dotted_as_name (COMMA dotted_as_name)* - -> ^(Import dotted_as_name+) - | 'from' dotted_name 'import' +//import_stmt: import_name | import_from +import_stmt : import_name + | import_from + ; + +//import_name: 'import' dotted_as_names +import_name : 'import' dotted_as_names + -> ^(Import dotted_as_names) + ; + +//XXX: needs work? +//import_from: ('from' ('.'* dotted_name | '.'+) +// 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_from: 'from' (DOT* dotted_name | DOT+) 'import' (STAR -> ^(ImportFrom dotted_name ^(Import STAR)) - | import_as_name (COMMA import_as_name)* - -> ^(ImportFrom dotted_name ^(Import import_as_name+)) + | import_as_names + -> ^(ImportFrom dotted_name ^(Import import_as_names)) + | LPAREN import_as_names RPAREN + -> ^(ImportFrom dotted_name ^(Import import_as_names)) ) - ; + ; -import_as_name : NAME ('as' NAME)? +//import_as_name: NAME [('as' | NAME) NAME] +import_as_name : name=NAME ('as' asname=NAME)? + -> ^(Alias $name ^(Asname $asname)?) ; -dotted_as_name : dotted_name ('as' NAME)? +//dotted_as_name: dotted_name [('as' | NAME) NAME] +dotted_as_name : dotted_name ('as' asname=NAME)? + -> ^(Alias dotted_name ^(Asname NAME)?) ; +//import_as_names: import_as_name (',' import_as_name)* [','] +import_as_names : import_as_name (COMMA! import_as_name)* (COMMA!)? + ; + +//dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_as_names : dotted_as_name (COMMA! dotted_as_name)* + ; +//dotted_name: NAME ('.' NAME)* dotted_name : NAME (DOT NAME)* ; +//global_stmt: 'global' NAME (',' NAME)* global_stmt : 'global' NAME (COMMA NAME)* -> ^(Global NAME+) ; +//exec_stmt: 'exec' expr ['in' test [',' test]] exec_stmt : 'exec' expr ('in' t1=test (COMMA t2=test)?)? -> ^(Exec expr ^(Globals $t1)? ^(Locals $t2)?) ; +//assert_stmt: 'assert' test [',' test] assert_stmt : 'assert' t1=test (COMMA t2=test)? -> ^(Assert ^(Test $t1) ^(Msg $t2)?) ; +//compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef compound_stmt : if_stmt | while_stmt | for_stmt | try_stmt + | with_stmt | funcdef | classdef ; +//if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] if_stmt: 'if' test COLON ifsuite=suite elif_clause* ('else' COLON elsesuite=suite)? -> ^(If test $ifsuite elif_clause* ^(Else $elsesuite)?) ; +//not in CPython's Grammar file elif_clause : 'elif' test COLON suite -> ^(Elif test suite) ; +//while_stmt: 'while' test ':' suite ['else' ':' suite] while_stmt : 'while' test COLON s1=suite ('else' COLON s2=suite)? -> ^(While test ^(Body $s1) ^(Else $s2)?) ; +//for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] for_stmt : 'for' exprlist 'in' testlist COLON s1=suite ('else' COLON s2=suite)? -> ^(For exprlist ^(In testlist) ^(Body $s1) ^(Else $s2)?) ; +//try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break +// ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite) try_stmt : 'try' COLON trysuite=suite ( (except_clause+ ('else' COLON elsesuite=suite)? -> ^(TryExcept ^(Body $trysuite) except_clause+ ^(Else $elsesuite)?)) @@ -349,25 +424,45 @@ ) ; +//with_stmt: 'with' test [ with_var ] ':' suite +with_stmt: 'with' test (with_var)? COLON suite + -> ^(With test with_var? ^(Body suite)) + ; + +//with_var: ('as' | NAME) expr +with_var: ('as' | NAME) expr + ; + +//except_clause: 'except' [test [',' test]] except_clause : 'except' (t1=test (COMMA t2=test)?)? COLON suite -> ^(Except ^(Type $t1)? ^(Name $t2)? ^(Body suite)) ; +//suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT suite : simple_stmt -> ^(Stmt simple_stmt) | NEWLINE! INDENT (stmt)+ DEDENT ; -test : and_test ('or'^ and_test)* - | lambdef +//FIXME: looks like this one is going to be tough. +//test: or_test ['if' or_test 'else' test] | lambdef +test: or_test //('if' test 'else' test)? + | lambdef + ; + +//or_test: and_test ('or' and_test)* +or_test : and_test ('or'^ and_test)* ; +//and_test: not_test ('and' not_test)* and_test : not_test ('and'^ not_test)* ; +//not_test: 'not' not_test | comparison not_test : 'not'^ not_test | comparison ; +//comparison: expr (comp_op expr)* comparison: expr (comp_op^ expr)* ; @@ -377,6 +472,7 @@ // ) // ; +//comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' comp_op : LESS | GREATER | EQUAL @@ -390,15 +486,19 @@ | 'is' 'not' -> IsNot ; +//expr: xor_expr ('|' xor_expr)* expr : xor_expr (VBAR^ xor_expr)* ; +//xor_expr: and_expr ('^' and_expr)* xor_expr : and_expr (CIRCUMFLEX^ and_expr)* ; +//and_expr: shift_expr ('&' shift_expr)* and_expr : shift_expr (AMPER^ shift_expr)* ; +//shift_expr: arith_expr (('<<'|'>>') arith_expr)* shift_expr : arith_expr ((LEFTSHIFT^|RIGHTSHIFT^) arith_expr)* ; @@ -410,23 +510,32 @@ // ; +//arith_expr: term (('+'|'-') term)* arith_expr: term ((PLUS^|MINUS^) term)* ; +//term: factor (('*'|'/'|'%'|'//') factor)* term : factor ((STAR^ | SLASH^ | PERCENT^ | DOUBLESLASH^ ) factor)* ; +//factor: ('+'|'-'|'~') factor | power factor : PLUS factor -> ^(UnaryPlus factor) | MINUS factor -> ^(UnaryMinus factor) | TILDE factor -> ^(UnaryTilde factor) | power ; -power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR factor)? - -> atom (trailer)* (DOUBLESTAR factor)? +//power: atom trailer* ['**' factor] +power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR^ factor)? ; -atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) +//atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ +atom : LPAREN + ( yield_expr -> ^(Parens yield_expr) + | testlist_gexp -> ^(Parens testlist_gexp?) + | -> ^(Parens) + ) + RPAREN | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) @@ -438,51 +547,72 @@ | (STRING)+ -> ^(String STRING+) ; +//listmaker: test ( list_for | (',' test)* [','] ) listmaker : test ( list_for -> ^(ListComp list_for) | (options {greedy=true;}:COMMA test)* -> test+ ) (COMMA)? ; +//testlist_gexp: test ( gen_for | (',' test)* [','] ) +testlist_gexp : test ( gen_for -> ^(GenExpFor gen_for) + | (options {k=2;}: COMMA test)* -> test+ + ) + (COMMA)? + ; + + +//lambdef: 'lambda' [varargslist] ':' test lambdef: 'lambda' (varargslist)? COLON test -> ^(Lambda varargslist? ^(Body test)) ; +//trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME trailer : LPAREN (arglist)? RPAREN -> ^(ArgList arglist?) | LBRACK subscriptlist RBRACK -> ^(SubscriptList subscriptlist) | DOT NAME ; +//subscriptlist: subscript (',' subscript)* [','] subscriptlist : subscript (options {greedy=true;}:COMMA subscript)* (COMMA)? -> subscript+ ; -//XXX: ^Subscript needs a better name +//subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] subscript : DOT DOT DOT -> Ellipsis | t1=test (COLON (t2=test)? (sliceop)?)? -> ^(Subscript ^(Start $t1) ^(End $t2)? ^(SliceOp sliceop)?) | COLON (test)? (sliceop)? -> ^(Subscript ^(End test)? ^(SliceOp sliceop)?) ; +//sliceop: ':' [test] sliceop : COLON (test)? -> test? ; +//exprlist: expr (',' expr)* [','] exprlist : expr (options {k=2;}: COMMA expr)* (COMMA)? -> ^(ExprList expr+) ; +//testlist: test (',' test)* [','] testlist : test (options {k=2;}: COMMA test)* (COMMA)? -> test+ ; +//XXX: +//testlist_safe: test [(',' test)+ [',']] + +//dictmaker: test ':' test (',' test ':' test)* [','] dictmaker : test COLON test (options {k=2;}:COMMA test COLON test)* (COMMA)? -> test+ ; +//classdef: 'class' NAME ['(' testlist ')'] ':' suite classdef: 'class' NAME (LPAREN testlist RPAREN)? COLON suite -> ^(ClassDef ^(Name NAME) ^(Bases testlist)? ^(Body suite)) ; +//arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) arglist : argument (COMMA argument)* ( COMMA ( STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? @@ -496,21 +626,46 @@ -> ^(KWArgs $kwargs) ; +//argument: [test '='] test // Really [keyword '='] test argument : t1=test (ASSIGN t2=test)? -> ^(Arg $t1 ^(Default $t2)?) ; +//list_iter: list_for | list_if list_iter : list_for | list_if ; +//list_for: 'for' exprlist 'in' testlist_safe [list_iter] list_for : 'for' exprlist 'in' testlist (list_iter)? ; +//list_if: 'if' test [list_iter] list_if : 'if' test (list_iter)? ; +//gen_iter: gen_for | gen_if +gen_iter: gen_for + | gen_if + ; +//gen_for: 'for' exprlist 'in' or_test [gen_iter] +gen_for: 'for' exprlist 'in' or_test gen_iter? + ; + +//gen_if: 'if' old_test [gen_iter] +gen_if: 'if' test gen_iter? + ; + +//yield_expr: 'yield' [testlist] +yield_expr : 'yield' testlist? + -> ^(Yield testlist?) + ; + +//XXX: +//testlist1: test (',' test)* + + LPAREN : '(' {implicitLineJoiningLevel++;} ; RPAREN : ')' {implicitLineJoiningLevel--;} ; @@ -599,6 +754,8 @@ DOT : '.' ; +AT : '@' ; + FLOAT : '.' DIGITS (Exponent)? | DIGITS ('.' (DIGITS (Exponent)?)? | Exponent) Added: trunk/sandbox/wierzbicki/antlr/grammar/Python23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python23.g (rev 0) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python23.g 2007-11-21 02:54:33 UTC (rev 3710) @@ -0,0 +1,800 @@ +/* + [The 'BSD licence'] + Copyright (c) 2004 Terence Parr and Loring Craymer + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** Python 2.3.3 Grammar + * + * Terence Parr and Loring Craymer + * February 2004 + * + * Converted to ANTLR v3 November 2005 by Terence Parr. + * + * This grammar was derived automatically from the Python 2.3.3 + * parser grammar to get a syntactically correct ANTLR grammar + * for Python. Then Terence hand tweaked it to be semantically + * correct; i.e., removed lookahead issues etc... It is LL(1) + * except for the (sometimes optional) trailing commas and semi-colons. + * It needs two symbols of lookahead in this case. + * + * Starting with Loring's preliminary lexer for Python, I modified it + * to do my version of the whole nasty INDENT/DEDENT issue just so I + * could understand the problem better. This grammar requires + * PythonTokenStream.java to work. Also I used some rules from the + * semi-formal grammar on the web for Python (automatically + * translated to ANTLR format by an ANTLR grammar, naturally <grin>). + * The lexical rules for python are particularly nasty and it took me + * a long time to get it 'right'; i.e., think about it in the proper + * way. Resist changing the lexer unless you've used ANTLR a lot. ;) + * + * I (Terence) tested this by running it on the jython-2.1/Lib + * directory of 40k lines of Python. + * + * REQUIRES ANTLR v3 + * + * + * Baby step towards an antlr based Jython parser. + * Terence's Lexer is intact pretty much unchanged, the parser has + * been altered to produce an AST - the AST work started from tne newcompiler + * grammar from Jim Baker minus post-2.3 features. The current parsing + * and compiling strategy looks like this: + * + * Python source->Python.g->simple antlr AST->PythonWalker.g-> + * decorated AST (org/python/parser/ast/*)->CodeCompiler(ASM)->.class + * + * for a very limited set of functionality. + */ + +grammar Python; +options { + output=AST; +} + +tokens { + INDENT; + DEDENT; + + Test; + Msg; + Stmt; + Import; + ImportFrom; + Name; + Body; + ClassDef; + Bases; + FunctionDef; + Args; + StarArgs; + KWArgs; + Arg; + Arguments; + Assign; + Compare; + Expr; + ExprList; + Tuple; + List; + Dict; + If; + Else; + Elif; + While; + Pass; + Print; + TryExcept; + TryFinally; + Except; + For; + Return; + Yield; + String; + IsNot; + In; + NotIn; + Raise; + Type; + Inst; + Tback; + Global; + Exec;Globals;Locals; + Assert; + Ellipsis; + ListComp; + Lambda; + Repr; + BinOp; + ArgList; + Subscript; + SubscriptList; + Targets; + Values; + Start; + End; + SliceOp; + UnaryPlus; + UnaryMinus; + UnaryTilde; + Delete; + Default; + Parens; + Alias; + Asname; +} + +@header { +package org.python.antlr; +} + +@lexer::header { +package org.python.antlr; +} + +@lexer::members { +/** Handles context-sensitive lexing of implicit line joining such as + * the case where newline is ignored in cases like this: + * a = [3, + * 4] + */ +int implicitLineJoiningLevel = 0; +int startPos=-1; +} + +//single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +single_input : NEWLINE! + | simple_stmt + | compound_stmt NEWLINE! + ; + +//file_input: (NEWLINE | stmt)* ENDMARKER +file_input : (NEWLINE! | stmt)* + ; + +//eval_input: testlist NEWLINE* ENDMARKER +eval_input : (NEWLINE!)* testlist (NEWLINE!)* + ; + +//funcdef: 'def' NAME parameters ':' suite +funcdef : 'def' NAME parameters COLON suite + -> ^(FunctionDef ^(Name NAME) ^(Args parameters) ^(Body suite)) + ; + +//parameters: '(' [varargslist] ')' +parameters : LPAREN (varargslist)? RPAREN + -> (varargslist)? + ; + +//varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] +varargslist : defparameter (options {greedy=true;}:COMMA defparameter)* + (COMMA + ( STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? + | DOUBLESTAR kwargs=NAME + )? + )? + -> ^(Arguments defparameter+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + | STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? + -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? + | DOUBLESTAR kwargs=NAME + -> ^(KWArgs $kwargs) + ; + +//not in CPython's Grammar file +defparameter : fpdef (ASSIGN test)? + ; + +//fpdef: NAME | '(' fplist ')' +fpdef : NAME + | LPAREN! fplist RPAREN! + ; + +//fplist: fpdef (',' fpdef)* [','] +fplist : fpdef (options {greedy=true;}:COMMA fpdef)* (COMMA)? + -> fpdef+ + ; + +//stmt: simple_stmt | compound_stmt +stmt : simple_stmt + -> ^(Stmt simple_stmt) + | compound_stmt + -> ^(Stmt compound_stmt) + ; + +//simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? NEWLINE + -> small_stmt+ + ; + +//small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt +small_stmt : expr_stmt -> ^(Expr expr_stmt) + | print_stmt + | del_stmt + | pass_stmt + | flow_stmt + | import_stmt + | global_stmt + | exec_stmt + | assert_stmt + ; + +//expr_stmt: testlist (augassign testlist | ('=' testlist)*) +expr_stmt : lhs=testlist + ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + | ((ASSIGN rhs=testlist)+ -> ^(Assign ^(Targets $lhs) ^(Values $rhs))) + | -> $lhs + ) + ; + +//augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' +augassign : PLUSEQUAL + | MINUSEQUAL + | STAREQUAL + | SLASHEQUAL + | PERCENTEQUAL + | AMPEREQUAL + | VBAREQUAL + | CIRCUMFLEXEQUAL + | LEFTSHIFTEQUAL + | RIGHTSHIFTEQUAL + | DOUBLESTAREQUAL + | DOUBLESLASHEQUAL + ; + +//print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) +print_stmt : 'print' + ( testlist + | RIGHTSHIFT testlist + )? + -> ^(Print RIGHTSHIFT? testlist?) + ; + +//del_stmt: 'del' exprlist +del_stmt : 'del' exprlist + -> ^(Delete exprlist) + ; + +//pass_stmt: 'pass' +pass_stmt : 'pass' + -> Pass + ; + +//flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +flow_stmt : break_stmt + | continue_stmt + | return_stmt + | raise_stmt + | yield_stmt + ; + +//break_stmt: 'break' +break_stmt : 'break' + ; + +//continue_stmt: 'continue' +continue_stmt : 'continue' + ; + +//return_stmt: 'return' [testlist] +return_stmt : 'return' (testlist)? + -> ^(Return testlist?) + ; + +//yield_stmt: 'yield' testlist +yield_stmt : 'yield' testlist + -> ^(Yield testlist) + ; + +//raise_stmt: 'raise' [test [',' test [',' test]]] +raise_stmt: 'raise' (t1=test (COMMA t2=test (COMMA t3=test)?)?)? + -> ^(Raise ^(Type $t1)? ^(Inst $t2)? ^(Tback $t3)?) + ; + +//import_stmt: 'import' dotted_as_name (',' dotted_as_name)* | 'from' dotted_name 'import' ('*' | import_as_name (',' import_as_name)*) +import_stmt : 'import' dotted_as_name (COMMA dotted_as_name)* + -> ^(Import dotted_as_name+) + | 'from' dotted_name 'import' + (STAR + -> ^(ImportFrom dotted_name ^(Import STAR)) + | import_as_name (COMMA import_as_name)* + -> ^(ImportFrom dotted_name ^(Import import_as_name+)) + ) + ; + +//import_as_name: NAME [NAME NAME] +import_as_name : name=NAME ('as' asname=NAME)? + -> ^(Alias $name ^(Asname $asname)?) + ; + +//dotted_as_name: dotted_name [NAME NAME] +dotted_as_name : dotted_name ('as' asname=NAME)? + -> ^(Alias dotted_name ^(Asname NAME)?) + ; + +//dotted_name: NAME ('.' NAME)* +dotted_name : NAME (DOT NAME)* + ; + +//global_stmt: 'global' NAME (',' NAME)* +global_stmt : 'global' NAME (COMMA NAME)* + -> ^(Global NAME+) + ; + +//exec_stmt: 'exec' expr ['in' test [',' test]] +exec_stmt : 'exec' expr ('in' t1=test (COMMA t2=test)?)? + -> ^(Exec expr ^(Globals $t1)? ^(Locals $t2)?) + ; + +//assert_stmt: 'assert' test [',' test] +assert_stmt : 'assert' t1=test (COMMA t2=test)? + -> ^(Assert ^(Test $t1) ^(Msg $t2)?) + ; + +//compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef +compound_stmt : if_stmt + | while_stmt + | for_stmt + | try_stmt + | funcdef + | classdef + ; + +//if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +if_stmt: 'if' test COLON ifsuite=suite elif_clause* ('else' COLON elsesuite=suite)? + -> ^(If test $ifsuite elif_clause* ^(Else $elsesuite)?) + ; + +//not in CPython's Grammar file +elif_clause : 'elif' test COLON suite + -> ^(Elif test suite) + ; + +//while_stmt: 'while' test ':' suite ['else' ':' suite] +while_stmt : 'while' test COLON s1=suite ('else' COLON s2=suite)? + -> ^(While test ^(Body $s1) ^(Else $s2)?) + ; + +//for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +for_stmt : 'for' exprlist 'in' testlist COLON s1=suite ('else' COLON s2=suite)? + -> ^(For exprlist ^(In testlist) ^(Body $s1) ^(Else $s2)?) + ; + +//try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break +// ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite) +try_stmt : 'try' COLON trysuite=suite + ( (except_clause+ ('else' COLON elsesuite=suite)? + -> ^(TryExcept ^(Body $trysuite) except_clause+ ^(Else $elsesuite)?)) + | ('finally' COLON suite + -> ^(TryFinally suite)) + ) + ; + +//except_clause: 'except' [test [',' test]] +except_clause : 'except' (t1=test (COMMA t2=test)?)? COLON suite + -> ^(Except ^(Type $t1)? ^(Name $t2)? ^(Body suite)) + ; + +//suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +suite : simple_stmt -> ^(Stmt simple_stmt) + | NEWLINE! INDENT (stmt)+ DEDENT + ; + +//test: and_test ('or' and_test)* | lambdef +test : and_test ('or'^ and_test)* + | lambdef + ; + +//and_test: not_test ('and' not_test)* +and_test : not_test ('and'^ not_test)* + ; + +//not_test: 'not' not_test | comparison +not_test : 'not'^ not_test + | comparison + ; + +//comparison: expr (comp_op expr)* +comparison: expr (comp_op^ expr)* + ; + +//comparison : expr +// ( ((comp_op expr)+ -> ^(Compare expr (comp_op expr)+)) +// | expr +// ) +// ; + +//comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +comp_op : LESS + | GREATER + | EQUAL + | GREATEREQUAL + | LESSEQUAL + | ALT_NOTEQUAL + | NOTEQUAL + | 'in' + | 'not' 'in' -> NotIn + | 'is' + | 'is' 'not' -> IsNot + ; + +//expr: xor_expr ('|' xor_expr)* +expr : xor_expr (VBAR^ xor_expr)* + ; + +//xor_expr: and_expr ('^' and_expr)* +xor_expr : and_expr (CIRCUMFLEX^ and_expr)* + ; + +//and_expr: shift_expr ('&' shift_expr)* +and_expr : shift_expr (AMPER^ shift_expr)* + ; + +//shift_expr: arith_expr (('<<'|'>>') arith_expr)* +shift_expr : arith_expr ((LEFTSHIFT^|RIGHTSHIFT^) arith_expr)* + ; + +//arith_expr : lhs=term +// ( ((PLUS) term)+ -> ^(PLUS $lhs term+) +// | ((MINUS) term)+ -> ^(MINUS $lhs term+) +// | -> ^(Expr $lhs) +// ) +// ; + + +//arith_expr: term (('+'|'-') term)* +arith_expr: term ((PLUS^|MINUS^) term)* + ; + +//term: factor (('*'|'/'|'%'|'//') factor)* +term : factor ((STAR^ | SLASH^ | PERCENT^ | DOUBLESLASH^ ) factor)* + ; + +//factor: ('+'|'-'|'~') factor | power +factor : PLUS factor -> ^(UnaryPlus factor) + | MINUS factor -> ^(UnaryMinus factor) + | TILDE factor -> ^(UnaryTilde factor) + | power + ; + +//power: atom trailer* ['**' factor] +power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR^ factor)? + ; + +//atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ +atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) + | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) + | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) + | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) + | NAME + | INT + | LONGINT + | FLOAT + | COMPLEX + | (STRING)+ -> ^(String STRING+) + ; + +//listmaker: test ( list_for | (',' test)* [','] ) +listmaker : test + ( list_for -> ^(ListComp list_for) + | (options {greedy=true;}:COMMA test)* -> test+ + ) (COMMA)? + ; + +//lambdef: 'lambda' [varargslist] ':' test +lambdef: 'lambda' (varargslist)? COLON test + -> ^(Lambda varargslist? ^(Body test)) + ; + +//trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +trailer : LPAREN (arglist)? RPAREN -> ^(ArgList arglist?) + | LBRACK subscriptlist RBRACK -> ^(SubscriptList subscriptlist) + | DOT NAME + ; + +//subscriptlist: subscript (',' subscript)* [','] +subscriptlist : subscript (options {greedy=true;}:COMMA subscript)* (COMMA)? + -> subscript+ + ; + +//subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] +subscript : DOT DOT DOT -> Ellipsis + | t1=test (COLON (t2=test)? (sliceop)?)? -> ^(Subscript ^(Start $t1) ^(End $t2)? ^(SliceOp sliceop)?) + | COLON (test)? (sliceop)? -> ^(Subscript ^(End test)? ^(SliceOp sliceop)?) + ; + +//sliceop: ':' [test] +sliceop : COLON (test)? -> test? + ; + +//exprlist: expr (',' expr)* [','] +exprlist : expr (options {k=2;}: COMMA expr)* (COMMA)? + -> ^(ExprList expr+) + ; + +//testlist: test (',' test)* [','] +testlist : test (options {k=2;}: COMMA test)* (COMMA)? + -> test+ + ; + +//XXX: +//testlist_safe: test [(',' test)+ [',']] + +//dictmaker: test ':' test (',' test ':' test)* [','] +dictmaker : test COLON test + (options {k=2;}:COMMA test COLON test)* (COMMA)? + -> test+ + ; + +//classdef: 'class' NAME ['(' testlist ')'] ':' suite +classdef: 'class' NAME (LPAREN testlist RPAREN)? COLON suite + -> ^(ClassDef ^(Name NAME) ^(Bases testlist)? ^(Body suite)) + ; + +//arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) +arglist : argument (COMMA argument)* + ( COMMA + ( STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? + | DOUBLESTAR kwargs=test + )? + )? + -> ^(Arguments argument+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + | STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? + -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? + | DOUBLESTAR kwargs=test + -> ^(KWArgs $kwargs) + ; + +//argument: [test '='] test // Really [keyword '='] test +argument : t1=test (ASSIGN t2=test)? + -> ^(Arg $t1 ^(Default $t2)?) + ; + +//list_iter: list_for | list_if +list_iter : list_for + | list_if + ; + +//list_for: 'for' exprlist 'in' testlist_safe [list_iter] +list_for : 'for' exprlist 'in' testlist (list_iter)? + ; + +//list_if: 'if' test [list_iter] +list_if : 'if' test (list_iter)? + ; + +//XXX: +//testlist1: test (',' test)* + + +LPAREN : '(' {implicitLineJoiningLevel++;} ; + +RPAREN : ')' {implicitLineJoiningLevel--;} ; + +LBRACK : '[' {implicitLineJoiningLevel++;} ; + +RBRACK : ']' {implicitLineJoiningLevel--;} ; + +COLON : ':' ; + +COMMA : ',' ; + +SEMI : ';' ; + +PLUS : '+' ; + +MINUS : '-' ; + +STAR : '*' ; + +SLASH : '/' ; + +VBAR : '|' ; + +AMPER : '&' ; + +LESS : '<' ; + +GREATER : '>' ; + +ASSIGN : '=' ; + +PERCENT : '%' ; + +BACKQUOTE : '`' ; + +LCURLY : '{' {implicitLineJoiningLevel++;} ; + +RCURLY : '}' {implicitLineJoiningLevel--;} ; + +CIRCUMFLEX : '^' ; + +TILDE : '~' ; + +EQUAL : '==' ; + +NOTEQUAL : '!=' ; + +ALT_NOTEQUAL: '<>' ; + +LESSEQUAL : '<=' ; + +LEFTSHIFT : '<<' ; + +GREATEREQUAL : '>=' ; + +RIGHTSHIFT : '>>' ; + +PLUSEQUAL : '+=' ; + +MINUSEQUAL : '-=' ; + +DOUBLESTAR : '**' ; + +STAREQUAL : '*=' ; + +DOUBLESLASH : '//' ; + +SLASHEQUAL : '/=' ; + +VBAREQUAL : '|=' ; + +PERCENTEQUAL : '%=' ; + +AMPEREQUAL : '&=' ; + +CIRCUMFLEXEQUAL : '^=' ; + +LEFTSHIFTEQUAL : '<<=' ; + +RIGHTSHIFTEQUAL : '>>=' ; + +DOUBLESTAREQUAL : '**=' ; + +DOUBLESLASHEQUAL : '//=' ; + +DOT : '.' ; + +FLOAT + : '.' DIGITS (Exponent)? + | DIGITS ('.' (DIGITS (Exponent)?)? | Exponent) + ; + +LONGINT + : INT ('l'|'L') + ; + +fragment +Exponent + : ('e' | 'E') ( '+' | '-' )? DIGITS + ; + +INT : // Hex + '0' ('x' | 'X') ( '0' .. '9' | 'a' .. 'f' | 'A' .. 'F' )+ + | // Octal + '0' DIGITS* + | '1'..'9' DIGITS* + ; + +COMPLEX + : INT ('j'|'J') + | FLOAT ('j'|'J') + ; + +fragment +DIGITS : ( '0' .. '9' )+ ; + +NAME: ( 'a' .. 'z' | 'A' .. 'Z' | '_') + ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '0' .. '9' )* + ; + +/** Match various string types. Note that greedy=false implies ''' + * should make us exit loop not continue. + */ +STRING + : ('r'|'u'|'ur')? + ( '\'\'\'' (options {greedy=false;}:.)* '\'\'\'' + | '"""' (options {greedy=false;}:.)* '"""' + | '"' (ESC|~('\\'|'\n'|'"'))* '"' + | '\'' (ESC|~('\\'|'\n'|'\''))* '\'' + ) + ; + +fragment +ESC + : '\\' . + ; + +/** Consume a newline and any whitespace at start of next line */ +CONTINUED_LINE + : '\\' ('\r')? '\n' (' '|'\t')* { $channel=HIDDEN; } + ; + +/** Treat a sequence of blank lines as a single blank line. If + * nested within a (..), {..}, or [..], then ignore newlines. + * If the first newline starts in column one, they are to be ignored. + */ +NEWLINE + : (('\r')? '\n' )+ + {if ( startPos==0 || implicitLineJoiningLevel>0 ) + $channel=HIDDEN; + } + ; + +WS : {startPos>0}?=> (' '|'\t')+ {$channel=HIDDEN;} + ; + +/** Grab everything before a real symbol. Then if newline, kill it + * as this is a blank line. If whitespace followed by comment, kill it + * as it's a comment on a line by itself. + * + * Ignore leading whitespace when nested in [..], (..), {..}. + */ +LEADING_WS +@init { + int spaces = 0; +} + : {startPos==0}?=> + ( {implicitLineJoiningLevel>0}? ( ' ' | '\t' )+ {$channel=HIDDEN;} + | ( ' ' { spaces++; } + | '\t' { spaces += 8; spaces -= (spaces \% 8); } + )+ + { + // make a string of n spaces where n is column number - 1 + char[] indentation = new char[spaces]; + for (int i=0; i<spaces; i++) { + indentation[i] = ' '; + } + String s = new String(indentation); + emit(new ClassicToken(LEADING_WS,new String(indentation))); + } + // kill trailing newline if present and then ignore + ( ('\r')? '\n' {if (token!=null) token.setChannel(HIDDEN); else $channel=HIDDEN;})* + // {token.setChannel(99); } + ) + ; + +/** Comments not on line by themselves are turned into newlines. + + b = a # end of line comment + + or + + a = [1, # weird + 2] + + This rule is invoked directly by nextToken when the comment is in + first column or when comment is on end of nonwhitespace line. + + Only match \n here if we didn't start on left edge; let NEWLINE return that. + Kill if newlines if we live on a line by ourselves + + Consume any leading whitespace if it starts on left edge. + */ +COMMENT +@init { + $channel=HIDDEN; +} + : {startPos==0}?=> (' '|'\t')* '#' (~'\n')* '\n'+ + | {startPos>0}?=> '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#' + ; Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-20 23:59:51 UTC (rev 3709) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-21 02:54:33 UTC (rev 3710) @@ -15,12 +15,15 @@ } ; +decorator: ^(Decorator dotted_name ^(ArgList arglist?)) + ; -funcdef - : ^(FunctionDef ^(Name NAME) ^(Args varargslist?) ^(Body suite)) { - } - ; +decorators: decorator+ + ; +funcdef : ^(FunctionDef (^(Decorators decorators))? ^(Name NAME) ^(Args varargslist?) ^(Body suite)) + ; + varargslist : ^(Arguments defparameter*) (^(StarArgs sname=NAME))? (^(KWArgs kname=NAME))? { } @@ -49,12 +52,13 @@ | global_stmt | exec_stmt | assert_stmt - | funcdef - | classdef | if_stmt | while_stmt | for_stmt | try_stmt + | with_stmt + | funcdef + | classdef ; expr_stmt @@ -110,7 +114,7 @@ return_stmt : ^(Return (testlist)?) ; -yield_stmt : 'yield' testlist +yield_stmt : ^(Yield testlist?) ; raise_stmt: ^(Raise (^(Type test))? (^(Inst test))? (^(Tback test))?) @@ -123,13 +127,11 @@ | ^(ImportFrom dotted_name ^(Import import_as_name+)) ; -import_as_name : name=NAME ('as' alias=NAME)? +import_as_name : ^(Alias NAME (^(Asname NAME))?) ; -dotted_as_name - : dotted_name ('as' NAME)? { - } - ; +dotted_as_name : ^(Alias dotted_name (^(Asname NAME))?) + ; dotted_name : start=NAME (DOT NAME)* { @@ -164,6 +166,13 @@ except_clause : ^(Except (^(Type test))? (^(Name test))? ^(Body suite)) ; + +with_stmt: ^(With test with_var? ^(Body suite)) + ; + +with_var: ('as' | NAME) test + ; + suite : INDENT stmt+ DEDENT @@ -180,24 +189,18 @@ | ^(MINUS left=test right=test) {} | ^(AMPER test test) | ^(VBAR test test) + | ^(CIRCUMFLEX test test) | ^(LEFTSHIFT test test) | ^(RIGHTSHIFT test test) + | ^(STAR test test) + | ^(SLASH test test) + | ^(PERCENT test test) + | ^(DOUBLESLASH test test) + | ^(DOUBLESTAR test test) + | ^(UnaryPlus test) + | ^(UnaryMinus test) + | ^(UnaryTilde test) | lambdef - | term - ; - -term - : ^(STAR factor factor) - | ^(SLASH factor factor) - | ^(PERCENT factor factor) - | ^(DOUBLESLASH factor factor) - | factor - ; - -factor - : ^(UnaryPlus factor) - | ^(UnaryMinus factor) - | ^(UnaryTilde factor) | atom (trailer)* {} ; @@ -219,6 +222,7 @@ atom : ^(List testlist?) {} | ^(ListComp list_for) {} + | ^(GenExpFor gen_for) {} | ^(Parens testlist?) {} | ^(Dict testlist?) {} | ^(Repr testlist?) {} @@ -288,3 +292,13 @@ list_if: 'if' test (list_iter)? ; +gen_iter: gen_for + | gen_if + ; + +gen_for: 'for' exprlist 'in' test gen_iter? + ; + +gen_if: 'if' test gen_iter? + ; + Added: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g (rev 0) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g 2007-11-21 02:54:33 UTC (rev 3710) @@ -0,0 +1,282 @@ +tree grammar PythonWalker; + +options { + tokenVocab=Python; + ASTLabelType=CommonTree; +} + +@header { +package org.python.antlr; +} + +module + : stmt* { + //System.out.println("<STMT>"); + } + ; + + +funcdef + : ^(FunctionDef ^(Name NAME) ^(Args varargslist?) ^(Body suite)) { + } + ; + +varargslist + : ^(Arguments defparameter*) (^(StarArgs sname=NAME))? (^(KWArgs kname=NAME))? { + } + | ^(StarArgs sname=NAME) (^(KWArgs kname=NAME))? { + } + | ^(KWArgs NAME) { + } + ; + +defparameter + : NAME (ASSIGN test )? { + } + ; + +stmt: + ^(Stmt any_stmt+) + ; + +any_stmt //combines simple_stmt and compound_stmt from Python.g + : expr_stmt + | print_stmt + | del_stmt + | pass_stmt + | flow_stmt + | import_stmt + | global_stmt + | exec_stmt + | assert_stmt + | funcdef + | classdef + | if_stmt + | while_stmt + | for_stmt + | try_stmt + ; + +expr_stmt + : ^(Expr testlist) { + } + | ^(Expr ^(augassign targ=test value=test)) { + } + | ^(Expr ^(Assign ^(Targets targs=testlist) ^(Values values=testlist))) { + } + ; + +augassign + : PLUSEQUAL {} + | MINUSEQUAL {} + | STAREQUAL {} + | SLASHEQUAL {} + | PERCENTEQUAL {} + | AMPEREQUAL {} + | VBAREQUAL {} + | CIRCUMFLEXEQUAL {} + | LEFTSHIFTEQUAL {} + | RIGHTSHIFTEQUAL {} + | DOUBLESTAREQUAL {} + | DOUBLESLASHEQUAL {} + ; + +print_stmt + : ^(Print RIGHTSHIFT? testlist?) + { + } + ; + +del_stmt : ^(Delete exprlist) + ; + +pass_stmt : Pass { + } + ; + +flow_stmt : break_stmt + | continue_stmt + | return_stmt + | raise_stmt + | yield_stmt + ; + +break_stmt : 'break' + ; + +continue_stmt : 'continue' + ; + +return_stmt : ^(Return (testlist)?) + ; + +yield_stmt : ^(Yield testlist) + ; + +raise_stmt: ^(Raise (^(Type test))? (^(Inst test))? (^(Tback test))?) + ; + +import_stmt + : ^(Import dotted_as_name+) { + } + | ^(ImportFrom dotted_name ^(Import STAR)) + | ^(ImportFrom dotted_name ^(Import import_as_name+)) + ; + +import_as_name : ^(Alias NAME (^(Asname NAME))?) + ; + +dotted_as_name : ^(Alias dotted_name (^(Asname NAME))?) + ; + +dotted_name + : start=NAME (DOT NAME)* { + } + ; + +global_stmt : ^(Global NAME+) + ; + +exec_stmt : ^(Exec test (^(Globals test))? (^(Locals test))?) + ; + +assert_stmt : ^(Assert ^(Test test) (^(Msg test))?) + ; + + +if_stmt: ^(If test suite elif_clause* (^(Else suite))?) + ; + +elif_clause : ^(Elif test suite) + ; + +while_stmt : ^(While test ^(Body suite) (^(Else suite))?) + ; + +for_stmt : ^(For exprlist ^(In testlist) ^(Body suite) (^(Else suite))?) + ; + +try_stmt : ^(TryExcept ^(Body suite) except_clause+ (^(Else suite))?) + | ^(TryFinally suite) + ; + +except_clause : ^(Except (^(Type test))? (^(Name test))? ^(Body suite)) + ; + +suite + : INDENT stmt+ DEDENT + | stmt+ + ; + +//FIXME: lots of placeholders +test + : ^('and' test test) {} + | ^('or' test test) {} + | ^('not' test) {} + | ^(comp_op left=test targs=test) + | ^(PLUS test test) + | ^(MINUS left=test right=test) {} + | ^(AMPER test test) + | ^(VBAR test test) + | ^(CIRCUMFLEX test test) + | ^(LEFTSHIFT test test) + | ^(RIGHTSHIFT test test) + | ^(STAR test test) + | ^(SLASH test test) + | ^(PERCENT test test) + | ^(DOUBLESLASH test test) + | ^(DOUBLESTAR test test) + | ^(UnaryPlus test) + | ^(UnaryMinus test) + | ^(UnaryTilde test) + | lambdef + | atom (trailer)* {} + ; + +comp_op + : LESS {} + | GREATER {} + | EQUAL {} + | GREATEREQUAL {} + | LESSEQUAL {} + | ALT_NOTEQUAL {} + | NOTEQUAL {} + | 'in' {} + | NotIn {} + | 'is' {} + | IsNot {} + ; + +//FIXME: lots of placeholders +atom + : ^(List testlist?) {} + | ^(ListComp list_for) {} + | ^(Parens testlist?) {} + | ^(Dict testlist?) {} + | ^(Repr testlist?) {} + | NAME {} + | INT {} + | LONGINT {} + | FLOAT {} + | COMPLEX {} + | stringlist { + } + ; + +stringlist + : ^(String string+) {} + ; + +string + : STRING {} + ; + +lambdef: ^(Lambda varargslist? ^(Body test)) + ; + +trailer + : ^(ArgList arglist?) + | ^(SubscriptList subscriptlist) + | DOT NAME + ; + +subscriptlist + : subscript+ + ; + +subscript : Ellipsis + | ^(Subscript (^(Start test))? (^(End test))? (^(SliceOp test?))?) + ; + +exprlist : ^(ExprList test+) + ; + +testlist + : test+ {} + ; + +classdef + : ^(ClassDef ^(Name NAME) (^(Bases testlist))? ^(Body suite)) { + //System.out.println("class "); + } + ; + +arglist + : ^(Arguments argument+) (^(StarArgs test))? (^(KWArgs test))? + | ^(StarArgs test) (^(KWArgs test))? + | ^(KWArgs test) + ; + +argument : ^(Arg test (^(Default test))?) + ; + +list_iter: list_for + | list_if + ; + +list_for: 'for' exprlist 'in' testlist (list_iter)? + ; + +list_if: 'if' test (list_iter)? + ; + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-25 15:45:58
|
Revision: 3718 http://jython.svn.sourceforge.net/jython/?rev=3718&view=rev Author: fwierzbicki Date: 2007-11-25 07:45:53 -0800 (Sun, 25 Nov 2007) Log Message: ----------- placeholders for 25 work. Added Paths: ----------- trunk/sandbox/wierzbicki/antlr/grammar/Python25.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker25.g Added: trunk/sandbox/wierzbicki/antlr/grammar/Python25.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python25.g (rev 0) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python25.g 2007-11-25 15:45:53 UTC (rev 3718) @@ -0,0 +1,881 @@ +/* + [The 'BSD licence'] + Copyright (c) 2004 Terence Parr and Loring Craymer + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/** Python 2.3.3 Grammar + * + * Terence Parr and Loring Craymer + * February 2004 + * + * Converted to ANTLR v3 November 2005 by Terence Parr. + * + * This grammar was derived automatically from the Python 2.3.3 + * parser grammar to get a syntactically correct ANTLR grammar + * for Python. Then Terence hand tweaked it to be semantically + * correct; i.e., removed lookahead issues etc... It is LL(1) + * except for the (sometimes optional) trailing commas and semi-colons. + * It needs two symbols of lookahead in this case. + * + * Starting with Loring's preliminary lexer for Python, I modified it + * to do my version of the whole nasty INDENT/DEDENT issue just so I + * could understand the problem better. This grammar requires + * PythonTokenStream.java to work. Also I used some rules from the + * semi-formal grammar on the web for Python (automatically + * translated to ANTLR format by an ANTLR grammar, naturally <grin>). + * The lexical rules for python are particularly nasty and it took me + * a long time to get it 'right'; i.e., think about it in the proper + * way. Resist changing the lexer unless you've used ANTLR a lot. ;) + * + * I (Terence) tested this by running it on the jython-2.1/Lib + * directory of 40k lines of Python. + * + * REQUIRES ANTLR v3 + * + * + * Baby step towards an antlr based Jython parser. + * Terence's Lexer is intact pretty much unchanged, the parser has + * been altered to produce an AST - the AST work started from tne newcompiler + * grammar from Jim Baker minus post-2.3 features. The current parsing + * and compiling strategy looks like this: + * + * Python source->Python.g->simple antlr AST->PythonWalker.g-> + * decorated AST (org/python/parser/ast/*)->CodeCompiler(ASM)->.class + * + * for a very limited set of functionality. + */ + +grammar Python; +options { + output=AST; +} + +tokens { + INDENT; + DEDENT; + + Test; + Msg; + Stmt; + Import; + ImportFrom; + Name; + Body; + ClassDef; + Bases; + FunctionDef; + Args; + StarArgs; + KWArgs; + Arg; + Arguments; + Assign; + Compare; + Expr; + ExprList; + Tuple; + List; + Dict; + If; + Else; + Elif; + While; + Pass; + Print; + TryExcept; + TryFinally; + Except; + For; + Return; + Yield; + String; + IsNot; + In; + NotIn; + Raise; + Type; + Inst; + Tback; + Global; + Exec;Globals;Locals; + Assert; + Ellipsis; + ListComp; + Lambda; + Repr; + BinOp; + ArgList; + Subscript; + SubscriptList; + Targets; + Values; + Start; + End; + SliceOp; + UnaryPlus; + UnaryMinus; + UnaryTilde; + Delete; + Default; + Parens; + Alias; + Asname; + Decorator; + Decorators; + With; + GenExpFor; +} + +@header { +package org.python.antlr; +} + +@lexer::header { +package org.python.antlr; +} + +@lexer::members { +/** Handles context-sensitive lexing of implicit line joining such as + * the case where newline is ignored in cases like this: + * a = [3, + * 4] + */ +int implicitLineJoiningLevel = 0; +int startPos=-1; +} + +//single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +single_input : NEWLINE! + | simple_stmt + | compound_stmt NEWLINE! + ; + +//file_input: (NEWLINE | stmt)* ENDMARKER +file_input : (NEWLINE! | stmt)* + ; + +//eval_input: testlist NEWLINE* ENDMARKER +eval_input : (NEWLINE!)* testlist (NEWLINE!)* + ; + +//decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorator: AT dotted_name (LPAREN arglist? RPAREN)? NEWLINE + -> ^(Decorator dotted_name ^(ArgList arglist?)) + ; + +//decorators: decorator+ +decorators: decorator+ + ; + +//funcdef: [decorators] 'def' NAME parameters ':' suite +funcdef : decorators? 'def' NAME parameters COLON suite + -> ^(FunctionDef ^(Name NAME) ^(Arguments parameters) ^(Body suite) ^(Decorators decorators)?) + ; + +//parameters: '(' [varargslist] ')' +parameters : LPAREN (varargslist)? RPAREN + -> (varargslist)? + ; + +//varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [','] +varargslist : defparameter (options {greedy=true;}:COMMA defparameter)* + (COMMA + ( STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? + | DOUBLESTAR kwargs=NAME + )? + )? + -> ^(Arguments defparameter+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + | STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)? + -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? + | DOUBLESTAR kwargs=NAME + -> ^(KWArgs $kwargs) + ; + +//not in CPython's Grammar file +defparameter : fpdef (ASSIGN test)? + ; + +//fpdef: NAME | '(' fplist ')' +fpdef : NAME + | LPAREN! fplist RPAREN! + ; + +//fplist: fpdef (',' fpdef)* [','] +fplist : fpdef (options {greedy=true;}:COMMA fpdef)* (COMMA)? + -> fpdef+ + ; + +//stmt: simple_stmt | compound_stmt +stmt : simple_stmt + -> ^(Stmt simple_stmt) + | compound_stmt + -> ^(Stmt compound_stmt) + ; + +//simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +simple_stmt : small_stmt (options {greedy=true;}:SEMI small_stmt)* (SEMI)? NEWLINE + -> small_stmt+ + ; + +//small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt +small_stmt : expr_stmt -> ^(Expr expr_stmt) + | print_stmt + | del_stmt + | pass_stmt + | flow_stmt + | import_stmt + | global_stmt + | exec_stmt + | assert_stmt + ; + +//expr_stmt: testlist (augassign testlist | ('=' testlist)*) +expr_stmt : lhs=testlist + ( (augassign yield_expr -> ^(augassign $lhs yield_expr)) + | (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + | ((ASSIGN yield_expr)+ -> ^(Assign ^(Targets $lhs) ^(Values yield_expr))) + | ((ASSIGN rhs=testlist)+ -> ^(Assign ^(Targets $lhs) ^(Values $rhs))) + | -> $lhs + ) + ; + +//augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' +augassign : PLUSEQUAL + | MINUSEQUAL + | STAREQUAL + | SLASHEQUAL + | PERCENTEQUAL + | AMPEREQUAL + | VBAREQUAL + | CIRCUMFLEXEQUAL + | LEFTSHIFTEQUAL + | RIGHTSHIFTEQUAL + | DOUBLESTAREQUAL + | DOUBLESLASHEQUAL + ; + +//print_stmt: 'print' ( [ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ] ) +print_stmt : 'print' + ( testlist + | RIGHTSHIFT testlist + )? + -> ^(Print RIGHTSHIFT? testlist?) + ; + +//del_stmt: 'del' exprlist +del_stmt : 'del' exprlist + -> ^(Delete exprlist) + ; + +//pass_stmt: 'pass' +pass_stmt : 'pass' + -> Pass + ; + +//flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +flow_stmt : break_stmt + | continue_stmt + | return_stmt + | raise_stmt + | yield_stmt + ; + +//break_stmt: 'break' +break_stmt : 'break' + ; + +//continue_stmt: 'continue' +continue_stmt : 'continue' + ; + +//return_stmt: 'return' [testlist] +return_stmt : 'return' (testlist)? + -> ^(Return testlist?) + ; + +//yield_stmt: yield_expr +yield_stmt : yield_expr + ; + +//raise_stmt: 'raise' [test [',' test [',' test]]] +raise_stmt: 'raise' (t1=test (COMMA t2=test (COMMA t3=test)?)?)? + -> ^(Raise ^(Type $t1)? ^(Inst $t2)? ^(Tback $t3)?) + ; + +//import_stmt: import_name | import_from +import_stmt : import_name + | import_from + ; + +//import_name: 'import' dotted_as_names +import_name : 'import' dotted_as_names + -> ^(Import dotted_as_names) + ; + +//XXX: needs work? +//import_from: ('from' ('.'* dotted_name | '.'+) +// 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_from: 'from' (DOT* dotted_name | DOT+) 'import' + (STAR + -> ^(ImportFrom dotted_name ^(Import STAR)) + | import_as_names + -> ^(ImportFrom dotted_name ^(Import import_as_names)) + | LPAREN import_as_names RPAREN + -> ^(ImportFrom dotted_name ^(Import import_as_names)) + ) + ; + +//import_as_name: NAME [('as' | NAME) NAME] +import_as_name : name=NAME ('as' asname=NAME)? + -> ^(Alias $name ^(Asname $asname)?) + ; + +//dotted_as_name: dotted_name [('as' | NAME) NAME] +dotted_as_name : dotted_name ('as' asname=NAME)? + -> ^(Alias dotted_name ^(Asname NAME)?) + ; + +//import_as_names: import_as_name (',' import_as_name)* [','] +import_as_names : import_as_name (COMMA! import_as_name)* (COMMA!)? + ; + +//dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_as_names : dotted_as_name (COMMA! dotted_as_name)* + ; +//dotted_name: NAME ('.' NAME)* +dotted_name : NAME (DOT NAME)* + ; + +//global_stmt: 'global' NAME (',' NAME)* +global_stmt : 'global' NAME (COMMA NAME)* + -> ^(Global NAME+) + ; + +//exec_stmt: 'exec' expr ['in' test [',' test]] +exec_stmt : 'exec' expr ('in' t1=test (COMMA t2=test)?)? + -> ^(Exec expr ^(Globals $t1)? ^(Locals $t2)?) + ; + +//assert_stmt: 'assert' test [',' test] +assert_stmt : 'assert' t1=test (COMMA t2=test)? + -> ^(Assert ^(Test $t1) ^(Msg $t2)?) + ; + +//compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef +compound_stmt : if_stmt + | while_stmt + | for_stmt + | try_stmt + | with_stmt + | funcdef + | classdef + ; + +//if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +if_stmt: 'if' test COLON ifsuite=suite elif_clause* ('else' COLON elsesuite=suite)? + -> ^(If test $ifsuite elif_clause* ^(Else $elsesuite)?) + ; + +//not in CPython's Grammar file +elif_clause : 'elif' test COLON suite + -> ^(Elif test suite) + ; + +//while_stmt: 'while' test ':' suite ['else' ':' suite] +while_stmt : 'while' test COLON s1=suite ('else' COLON s2=suite)? + -> ^(While test ^(Body $s1) ^(Else $s2)?) + ; + +//for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +for_stmt : 'for' exprlist 'in' testlist COLON s1=suite ('else' COLON s2=suite)? + -> ^(For exprlist ^(In testlist) ^(Body $s1) ^(Else $s2)?) + ; + +//try_stmt: ('try' ':' suite (except_clause ':' suite)+ #diagram:break +// ['else' ':' suite] | 'try' ':' suite 'finally' ':' suite) +try_stmt : 'try' COLON trysuite=suite + ( (except_clause+ ('else' COLON elsesuite=suite)? + -> ^(TryExcept ^(Body $trysuite) except_clause+ ^(Else $elsesuite)?)) + | ('finally' COLON suite + -> ^(TryFinally suite)) + ) + ; + +//with_stmt: 'with' test [ with_var ] ':' suite +with_stmt: 'with' test (with_var)? COLON suite + -> ^(With test with_var? ^(Body suite)) + ; + +//with_var: ('as' | NAME) expr +with_var: ('as' | NAME) expr + ; + +//except_clause: 'except' [test [',' test]] +except_clause : 'except' (t1=test (COMMA t2=test)?)? COLON suite + -> ^(Except ^(Type $t1)? ^(Name $t2)? ^(Body suite)) + ; + +//suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +suite : simple_stmt -> ^(Stmt simple_stmt) + | NEWLINE! INDENT (stmt)+ DEDENT + ; + +//FIXME: looks like this one is going to be tough. +//test: or_test ['if' or_test 'else' test] | lambdef +test: or_test //('if' test 'else' test)? + | lambdef + ; + +//or_test: and_test ('or' and_test)* +or_test : and_test ('or'^ and_test)* + ; + +//and_test: not_test ('and' not_test)* +and_test : not_test ('and'^ not_test)* + ; + +//not_test: 'not' not_test | comparison +not_test : 'not'^ not_test + | comparison + ; + +//comparison: expr (comp_op expr)* +comparison: expr (comp_op^ expr)* + ; + +//comparison : expr +// ( ((comp_op expr)+ -> ^(Compare expr (comp_op expr)+)) +// | expr +// ) +// ; + +//comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +comp_op : LESS + | GREATER + | EQUAL + | GREATEREQUAL + | LESSEQUAL + | ALT_NOTEQUAL + | NOTEQUAL + | 'in' + | 'not' 'in' -> NotIn + | 'is' + | 'is' 'not' -> IsNot + ; + +//expr: xor_expr ('|' xor_expr)* +expr : xor_expr (VBAR^ xor_expr)* + ; + +//xor_expr: and_expr ('^' and_expr)* +xor_expr : and_expr (CIRCUMFLEX^ and_expr)* + ; + +//and_expr: shift_expr ('&' shift_expr)* +and_expr : shift_expr (AMPER^ shift_expr)* + ; + +//shift_expr: arith_expr (('<<'|'>>') arith_expr)* +shift_expr : arith_expr ((LEFTSHIFT^|RIGHTSHIFT^) arith_expr)* + ; + +//arith_expr : lhs=term +// ( ((PLUS) term)+ -> ^(PLUS $lhs term+) +// | ((MINUS) term)+ -> ^(MINUS $lhs term+) +// | -> ^(Expr $lhs) +// ) +// ; + + +//arith_expr: term (('+'|'-') term)* +arith_expr: term ((PLUS^|MINUS^) term)* + ; + +//term: factor (('*'|'/'|'%'|'//') factor)* +term : factor ((STAR^ | SLASH^ | PERCENT^ | DOUBLESLASH^ ) factor)* + ; + +//factor: ('+'|'-'|'~') factor | power +factor : PLUS factor -> ^(UnaryPlus factor) + | MINUS factor -> ^(UnaryMinus factor) + | TILDE factor -> ^(UnaryTilde factor) + | power + ; + +//power: atom trailer* ['**' factor] +power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR^ factor)? + ; + +//atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ +atom : LPAREN + ( yield_expr -> ^(Parens yield_expr) + | testlist_gexp -> ^(Parens testlist_gexp?) + | -> ^(Parens) + ) + RPAREN + | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) + | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) + | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) + | NAME + | INT + | LONGINT + | FLOAT + | COMPLEX + | (STRING)+ -> ^(String STRING+) + ; + +//listmaker: test ( list_for | (',' test)* [','] ) +listmaker : test + ( list_for -> ^(ListComp list_for) + | (options {greedy=true;}:COMMA test)* -> test+ + ) (COMMA)? + ; + +//testlist_gexp: test ( gen_for | (',' test)* [','] ) +testlist_gexp : test ( gen_for -> ^(GenExpFor gen_for) + | (options {k=2;}: COMMA test)* -> test+ + ) + (COMMA)? + ; + + +//lambdef: 'lambda' [varargslist] ':' test +lambdef: 'lambda' (varargslist)? COLON test + -> ^(Lambda varargslist? ^(Body test)) + ; + +//trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +trailer : LPAREN (arglist)? RPAREN -> ^(ArgList arglist?) + | LBRACK subscriptlist RBRACK -> ^(SubscriptList subscriptlist) + | DOT NAME + ; + +//subscriptlist: subscript (',' subscript)* [','] +subscriptlist : subscript (options {greedy=true;}:COMMA subscript)* (COMMA)? + -> subscript+ + ; + +//subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] +subscript : DOT DOT DOT -> Ellipsis + | t1=test (COLON (t2=test)? (sliceop)?)? -> ^(Subscript ^(Start $t1) ^(End $t2)? ^(SliceOp sliceop)?) + | COLON (test)? (sliceop)? -> ^(Subscript ^(End test)? ^(SliceOp sliceop)?) + ; + +//sliceop: ':' [test] +sliceop : COLON (test)? -> test? + ; + +//exprlist: expr (',' expr)* [','] +exprlist : expr (options {k=2;}: COMMA expr)* (COMMA)? + -> ^(ExprList expr+) + ; + +//testlist: test (',' test)* [','] +testlist : test (options {k=2;}: COMMA test)* (COMMA)? + -> test+ + ; + +//XXX: +//testlist_safe: test [(',' test)+ [',']] + +//dictmaker: test ':' test (',' test ':' test)* [','] +dictmaker : test COLON test + (options {k=2;}:COMMA test COLON test)* (COMMA)? + -> test+ + ; + +//classdef: 'class' NAME ['(' testlist ')'] ':' suite +classdef: 'class' NAME (LPAREN testlist RPAREN)? COLON suite + -> ^(ClassDef ^(Name NAME) ^(Bases testlist)? ^(Body suite)) + ; + +//arglist: (argument ',')* (argument [',']| '*' test [',' '**' test] | '**' test) +arglist : argument (COMMA argument)* + ( COMMA + ( STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? + | DOUBLESTAR kwargs=test + )? + )? + -> ^(Arguments argument+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + | STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? + -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? + | DOUBLESTAR kwargs=test + -> ^(KWArgs $kwargs) + ; + +//argument: [test '='] test // Really [keyword '='] test +argument : t1=test (ASSIGN t2=test)? + -> ^(Arg $t1 ^(Default $t2)?) + ; + +//list_iter: list_for | list_if +list_iter : list_for + | list_if + ; + +//list_for: 'for' exprlist 'in' testlist_safe [list_iter] +list_for : 'for' exprlist 'in' testlist (list_iter)? + ; + +//list_if: 'if' test [list_iter] +list_if : 'if' test (list_iter)? + ; + +//gen_iter: gen_for | gen_if +gen_iter: gen_for + | gen_if + ; + +//gen_for: 'for' exprlist 'in' or_test [gen_iter] +gen_for: 'for' exprlist 'in' or_test gen_iter? + ; + +//gen_if: 'if' old_test [gen_iter] +gen_if: 'if' test gen_iter? + ; + +//yield_expr: 'yield' [testlist] +yield_expr : 'yield' testlist? + -> ^(Yield testlist?) + ; + +//XXX: +//testlist1: test (',' test)* + + +LPAREN : '(' {implicitLineJoiningLevel++;} ; + +RPAREN : ')' {implicitLineJoiningLevel--;} ; + +LBRACK : '[' {implicitLineJoiningLevel++;} ; + +RBRACK : ']' {implicitLineJoiningLevel--;} ; + +COLON : ':' ; + +COMMA : ',' ; + +SEMI : ';' ; + +PLUS : '+' ; + +MINUS : '-' ; + +STAR : '*' ; + +SLASH : '/' ; + +VBAR : '|' ; + +AMPER : '&' ; + +LESS : '<' ; + +GREATER : '>' ; + +ASSIGN : '=' ; + +PERCENT : '%' ; + +BACKQUOTE : '`' ; + +LCURLY : '{' {implicitLineJoiningLevel++;} ; + +RCURLY : '}' {implicitLineJoiningLevel--;} ; + +CIRCUMFLEX : '^' ; + +TILDE : '~' ; + +EQUAL : '==' ; + +NOTEQUAL : '!=' ; + +ALT_NOTEQUAL: '<>' ; + +LESSEQUAL : '<=' ; + +LEFTSHIFT : '<<' ; + +GREATEREQUAL : '>=' ; + +RIGHTSHIFT : '>>' ; + +PLUSEQUAL : '+=' ; + +MINUSEQUAL : '-=' ; + +DOUBLESTAR : '**' ; + +STAREQUAL : '*=' ; + +DOUBLESLASH : '//' ; + +SLASHEQUAL : '/=' ; + +VBAREQUAL : '|=' ; + +PERCENTEQUAL : '%=' ; + +AMPEREQUAL : '&=' ; + +CIRCUMFLEXEQUAL : '^=' ; + +LEFTSHIFTEQUAL : '<<=' ; + +RIGHTSHIFTEQUAL : '>>=' ; + +DOUBLESTAREQUAL : '**=' ; + +DOUBLESLASHEQUAL : '//=' ; + +DOT : '.' ; + +AT : '@' ; + +FLOAT + : '.' DIGITS (Exponent)? + | DIGITS ('.' (DIGITS (Exponent)?)? | Exponent) + ; + +LONGINT + : INT ('l'|'L') + ; + +fragment +Exponent + : ('e' | 'E') ( '+' | '-' )? DIGITS + ; + +INT : // Hex + '0' ('x' | 'X') ( '0' .. '9' | 'a' .. 'f' | 'A' .. 'F' )+ + | // Octal + '0' DIGITS* + | '1'..'9' DIGITS* + ; + +COMPLEX + : INT ('j'|'J') + | FLOAT ('j'|'J') + ; + +fragment +DIGITS : ( '0' .. '9' )+ ; + +NAME: ( 'a' .. 'z' | 'A' .. 'Z' | '_') + ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '0' .. '9' )* + ; + +/** Match various string types. Note that greedy=false implies ''' + * should make us exit loop not continue. + */ +STRING + : ('r'|'u'|'ur')? + ( '\'\'\'' (options {greedy=false;}:.)* '\'\'\'' + | '"""' (options {greedy=false;}:.)* '"""' + | '"' (ESC|~('\\'|'\n'|'"'))* '"' + | '\'' (ESC|~('\\'|'\n'|'\''))* '\'' + ) + ; + +fragment +ESC + : '\\' . + ; + +/** Consume a newline and any whitespace at start of next line */ +CONTINUED_LINE + : '\\' ('\r')? '\n' (' '|'\t')* { $channel=HIDDEN; } + ; + +/** Treat a sequence of blank lines as a single blank line. If + * nested within a (..), {..}, or [..], then ignore newlines. + * If the first newline starts in column one, they are to be ignored. + */ +NEWLINE + : (('\r')? '\n' )+ + {if ( startPos==0 || implicitLineJoiningLevel>0 ) + $channel=HIDDEN; + } + ; + +WS : {startPos>0}?=> (' '|'\t')+ {$channel=HIDDEN;} + ; + +/** Grab everything before a real symbol. Then if newline, kill it + * as this is a blank line. If whitespace followed by comment, kill it + * as it's a comment on a line by itself. + * + * Ignore leading whitespace when nested in [..], (..), {..}. + */ +LEADING_WS +@init { + int spaces = 0; +} + : {startPos==0}?=> + ( {implicitLineJoiningLevel>0}? ( ' ' | '\t' )+ {$channel=HIDDEN;} + | ( ' ' { spaces++; } + | '\t' { spaces += 8; spaces -= (spaces \% 8); } + )+ + { + // make a string of n spaces where n is column number - 1 + char[] indentation = new char[spaces]; + for (int i=0; i<spaces; i++) { + indentation[i] = ' '; + } + String s = new String(indentation); + emit(new ClassicToken(LEADING_WS,new String(indentation))); + } + // kill trailing newline if present and then ignore + ( ('\r')? '\n' {if (token!=null) token.setChannel(HIDDEN); else $channel=HIDDEN;})* + // {token.setChannel(99); } + ) + ; + +/** Comments not on line by themselves are turned into newlines. + + b = a # end of line comment + + or + + a = [1, # weird + 2] + + This rule is invoked directly by nextToken when the comment is in + first column or when comment is on end of nonwhitespace line. + + Only match \n here if we didn't start on left edge; let NEWLINE return that. + Kill if newlines if we live on a line by ourselves + + Consume any leading whitespace if it starts on left edge. + */ +COMMENT +@init { + $channel=HIDDEN; +} + : {startPos==0}?=> (' '|'\t')* '#' (~'\n')* '\n'+ + | {startPos>0}?=> '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#' + ; Added: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker25.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker25.g (rev 0) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker25.g 2007-11-25 15:45:53 UTC (rev 3718) @@ -0,0 +1,304 @@ +tree grammar PythonWalker; + +options { + tokenVocab=Python; + ASTLabelType=CommonTree; +} + +@header { +package org.python.antlr; +} + +module + : stmt* { + //System.out.println("<STMT>"); + } + ; + +decorator: ^(Decorator dotted_name ^(ArgList arglist?)) + ; + +decorators: decorator+ + ; + +funcdef : ^(FunctionDef (^(Decorators decorators))? ^(Name NAME) ^(Args varargslist?) ^(Body suite)) + ; + +varargslist + : ^(Arguments defparameter*) (^(StarArgs sname=NAME))? (^(KWArgs kname=NAME))? { + } + | ^(StarArgs sname=NAME) (^(KWArgs kname=NAME))? { + } + | ^(KWArgs NAME) { + } + ; + +defparameter + : NAME (ASSIGN test )? { + } + ; + +stmt: + ^(Stmt any_stmt+) + ; + +any_stmt //combines simple_stmt and compound_stmt from Python.g + : expr_stmt + | print_stmt + | del_stmt + | pass_stmt + | flow_stmt + | import_stmt + | global_stmt + | exec_stmt + | assert_stmt + | if_stmt + | while_stmt + | for_stmt + | try_stmt + | with_stmt + | funcdef + | classdef + ; + +expr_stmt + : ^(Expr testlist) { + } + | ^(Expr ^(augassign targ=test value=test)) { + } + | ^(Expr ^(Assign ^(Targets targs=testlist) ^(Values values=testlist))) { + } + ; + +augassign + : PLUSEQUAL {} + | MINUSEQUAL {} + | STAREQUAL {} + | SLASHEQUAL {} + | PERCENTEQUAL {} + | AMPEREQUAL {} + | VBAREQUAL {} + | CIRCUMFLEXEQUAL {} + | LEFTSHIFTEQUAL {} + | RIGHTSHIFTEQUAL {} + | DOUBLESTAREQUAL {} + | DOUBLESLASHEQUAL {} + ; + +print_stmt + : ^(Print RIGHTSHIFT? testlist?) + { + } + ; + +del_stmt : ^(Delete exprlist) + ; + +pass_stmt : Pass { + } + ; + +flow_stmt : break_stmt + | continue_stmt + | return_stmt + | raise_stmt + | yield_stmt + ; + +break_stmt : 'break' + ; + +continue_stmt : 'continue' + ; + +return_stmt : ^(Return (testlist)?) + ; + +yield_stmt : ^(Yield testlist?) + ; + +raise_stmt: ^(Raise (^(Type test))? (^(Inst test))? (^(Tback test))?) + ; + +import_stmt + : ^(Import dotted_as_name+) { + } + | ^(ImportFrom dotted_name ^(Import STAR)) + | ^(ImportFrom dotted_name ^(Import import_as_name+)) + ; + +import_as_name : ^(Alias NAME (^(Asname NAME))?) + ; + +dotted_as_name : ^(Alias dotted_name (^(Asname NAME))?) + ; + +dotted_name + : start=NAME (DOT NAME)* { + } + ; + +global_stmt : ^(Global NAME+) + ; + +exec_stmt : ^(Exec test (^(Globals test))? (^(Locals test))?) + ; + +assert_stmt : ^(Assert ^(Test test) (^(Msg test))?) + ; + + +if_stmt: ^(If test suite elif_clause* (^(Else suite))?) + ; + +elif_clause : ^(Elif test suite) + ; + +while_stmt : ^(While test ^(Body suite) (^(Else suite))?) + ; + +for_stmt : ^(For exprlist ^(In testlist) ^(Body suite) (^(Else suite))?) + ; + +try_stmt : ^(TryExcept ^(Body suite) except_clause+ (^(Else suite))?) + | ^(TryFinally suite) + ; + +except_clause : ^(Except (^(Type test))? (^(Name test))? ^(Body suite)) + ; + +with_stmt: ^(With test with_var? ^(Body suite)) + ; + +with_var: ('as' | NAME) test + ; + + +suite + : INDENT stmt+ DEDENT + | stmt+ + ; + +//FIXME: lots of placeholders +test + : ^('and' test test) {} + | ^('or' test test) {} + | ^('not' test) {} + | ^(comp_op left=test targs=test) + | ^(PLUS test test) + | ^(MINUS left=test right=test) {} + | ^(AMPER test test) + | ^(VBAR test test) + | ^(CIRCUMFLEX test test) + | ^(LEFTSHIFT test test) + | ^(RIGHTSHIFT test test) + | ^(STAR test test) + | ^(SLASH test test) + | ^(PERCENT test test) + | ^(DOUBLESLASH test test) + | ^(DOUBLESTAR test test) + | ^(UnaryPlus test) + | ^(UnaryMinus test) + | ^(UnaryTilde test) + | lambdef + | atom (trailer)* {} + ; + +comp_op + : LESS {} + | GREATER {} + | EQUAL {} + | GREATEREQUAL {} + | LESSEQUAL {} + | ALT_NOTEQUAL {} + | NOTEQUAL {} + | 'in' {} + | NotIn {} + | 'is' {} + | IsNot {} + ; + +//FIXME: lots of placeholders +atom + : ^(List testlist?) {} + | ^(ListComp list_for) {} + | ^(GenExpFor gen_for) {} + | ^(Parens testlist?) {} + | ^(Dict testlist?) {} + | ^(Repr testlist?) {} + | NAME {} + | INT {} + | LONGINT {} + | FLOAT {} + | COMPLEX {} + | stringlist { + } + ; + +stringlist + : ^(String string+) {} + ; + +string + : STRING {} + ; + +lambdef: ^(Lambda varargslist? ^(Body test)) + ; + +trailer + : ^(ArgList arglist?) + | ^(SubscriptList subscriptlist) + | DOT NAME + ; + +subscriptlist + : subscript+ + ; + +subscript : Ellipsis + | ^(Subscript (^(Start test))? (^(End test))? (^(SliceOp test?))?) + ; + +exprlist : ^(ExprList test+) + ; + +testlist + : test+ {} + ; + +classdef + : ^(ClassDef ^(Name NAME) (^(Bases testlist))? ^(Body suite)) { + //System.out.println("class "); + } + ; + +arglist + : ^(Arguments argument+) (^(StarArgs test))? (^(KWArgs test))? + | ^(StarArgs test) (^(KWArgs test))? + | ^(KWArgs test) + ; + +argument : ^(Arg test (^(Default test))?) + ; + +list_iter: list_for + | list_if + ; + +list_for: 'for' exprlist 'in' testlist (list_iter)? + ; + +list_if: 'if' test (list_iter)? + ; + +gen_iter: gen_for + | gen_if + ; + +gen_for: 'for' exprlist 'in' test gen_iter? + ; + +gen_if: 'if' test gen_iter? + ; + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-25 16:03:42
|
Revision: 3719 http://jython.svn.sourceforge.net/jython/?rev=3719&view=rev Author: fwierzbicki Date: 2007-11-25 08:03:40 -0800 (Sun, 25 Nov 2007) Log Message: ----------- Parens -> Tuple Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python23.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python23.g 2007-11-25 15:45:53 UTC (rev 3718) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python23.g 2007-11-25 16:03:40 UTC (rev 3719) @@ -140,7 +140,6 @@ UnaryTilde; Delete; Default; - Parens; Alias; Asname; } @@ -482,7 +481,7 @@ ; //atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ -atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) +atom : LPAREN (testlist)? RPAREN -> ^(Tuple testlist?) | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g 2007-11-25 15:45:53 UTC (rev 3718) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g 2007-11-25 16:03:40 UTC (rev 3719) @@ -211,7 +211,7 @@ atom : ^(List testlist?) {} | ^(ListComp list_for) {} - | ^(Parens testlist?) {} + | ^(Tuple testlist?) {} | ^(Dict testlist?) {} | ^(Repr testlist?) {} | NAME {} This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-26 02:42:41
|
Revision: 3728 http://jython.svn.sourceforge.net/jython/?rev=3728&view=rev Author: fwierzbicki Date: 2007-11-25 18:42:40 -0800 (Sun, 25 Nov 2007) Log Message: ----------- Better assign parsing and walking using antlr syntactic predicate. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/Python23.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-25 23:54:16 UTC (rev 3727) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-26 02:42:40 UTC (rev 3728) @@ -93,11 +93,11 @@ Arg; Arguments; Assign; - Assigns; Compare; Expr; ExprList; Tuple; + Parens;//do we need? List; Dict; If; @@ -131,8 +131,8 @@ ArgList; Subscript; SubscriptList; - Targets; - Values; + Target; + Value; Start; End; SliceOp; @@ -240,20 +240,22 @@ ; //expr_stmt: testlist (augassign testlist | ('=' testlist)*) -expr_stmt : lhs=testlist - ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) - | ((assigns) -> ^(Assigns ^(Assign $lhs) assigns)) - | -> $lhs - ) - ; +expr_stmt + : lhs=testlist + ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + | ((assigns) -> ^(Assign ^(Target $lhs) assigns)) + | -> $lhs + ) + ; //not in CPython's Grammar file assigns : assign+ ; //not in CPython's Grammar file -assign : ASSIGN testlist -> ^(Assign testlist) - ; +assign : (ASSIGN testlist ASSIGN) => ASSIGN testlist -> ^(Target testlist) + | ASSIGN testlist -> ^(Value testlist) + ; //augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' augassign : PLUSEQUAL @@ -490,7 +492,7 @@ ; //atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ -atom : LPAREN (testlist)? RPAREN -> ^(Tuple testlist?) +atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python23.g 2007-11-25 23:54:16 UTC (rev 3727) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python23.g 2007-11-26 02:42:40 UTC (rev 3728) @@ -93,6 +93,7 @@ Arg; Arguments; Assign; + Assigns; Compare; Expr; ExprList; @@ -131,7 +132,7 @@ Subscript; SubscriptList; Targets; - Values; + Value; Start; End; SliceOp; @@ -239,13 +240,26 @@ ; //expr_stmt: testlist (augassign testlist | ('=' testlist)*) -expr_stmt : lhs=testlist - ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) - | ((ASSIGN rhs=testlist)+ -> ^(Assign ^(Targets $lhs) ^(Values $rhs))) - | -> $lhs - ) - ; +expr_stmt +@after { + System.out.println($expr_stmt.text); +} + : lhs=testlist + ( (augassign rhs=testlist -> ^(augassign $lhs $rhs)) + | ((assigns) -> ^(Assigns ^(Assign $lhs) assigns)) + | -> $lhs + ) + ; +//not in CPython's Grammar file +assigns : assign+ + ; + +//not in CPython's Grammar file +assign : (ASSIGN testlist ASSIGN) => ASSIGN testlist -> ^(Assign testlist) + | ASSIGN testlist -> ^(Value testlist) + ; + //augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' augassign : PLUSEQUAL | MINUSEQUAL Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-25 23:54:16 UTC (rev 3727) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-26 02:42:40 UTC (rev 3728) @@ -62,7 +62,7 @@ } | ^(Expr ^(augassign targ=test value=test)) { } - | ^(Expr ^(Assigns (^(Assign testlist))+)) { + | ^(Expr ^(Assign (^(Target testlist))+ ^(Value testlist))) { } ; @@ -211,7 +211,7 @@ atom : ^(List testlist?) {} | ^(ListComp list_for) {} - | ^(Tuple testlist?) {} + | ^(Parens testlist?) {} | ^(Dict testlist?) {} | ^(Repr testlist?) {} | NAME {} Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g 2007-11-25 23:54:16 UTC (rev 3727) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker23.g 2007-11-26 02:42:40 UTC (rev 3728) @@ -62,7 +62,7 @@ } | ^(Expr ^(augassign targ=test value=test)) { } - | ^(Expr ^(Assign ^(Targets targs=testlist) ^(Values values=testlist))) { + | ^(Expr ^(Assign (^(Target testlist))+ ^(Value testlist))) { } ; @@ -211,7 +211,7 @@ atom : ^(List testlist?) {} | ^(ListComp list_for) {} - | ^(Tuple testlist?) {} + | ^(Parens testlist?) {} | ^(Dict testlist?) {} | ^(Repr testlist?) {} | NAME {} This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-28 00:30:53
|
Revision: 3738 http://jython.svn.sourceforge.net/jython/?rev=3738&view=rev Author: fwierzbicki Date: 2007-11-27 16:30:51 -0800 (Tue, 27 Nov 2007) Log Message: ----------- A bunch of improvements to the AST gen. Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-27 22:33:15 UTC (rev 3737) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-28 00:30:51 UTC (rev 3738) @@ -97,7 +97,6 @@ Expr; ExprList; Tuple; - Parens;//do we need? List; Dict; If; @@ -112,7 +111,7 @@ For; Return; Yield; - String; + Str; IsNot; In; NotIn; @@ -143,6 +142,12 @@ Default; Alias; Asname; + Num; + Module; + Call; + ListCompFor; + ListCompIf; + Iter; } @header { @@ -170,7 +175,7 @@ ; //file_input: (NEWLINE | stmt)* ENDMARKER -file_input : (NEWLINE! | stmt)* +file_input : (NEWLINE | stmt)* -> ^(Module ^(Body stmt*)) ; //eval_input: testlist NEWLINE* ENDMARKER @@ -206,7 +211,7 @@ ; //fpdef: NAME | '(' fplist ')' -fpdef : NAME +fpdef : NAME -> ^(Name NAME) | LPAREN! fplist RPAREN! ; @@ -217,9 +222,7 @@ //stmt: simple_stmt | compound_stmt stmt : simple_stmt - -> ^(Stmt simple_stmt) | compound_stmt - -> ^(Stmt compound_stmt) ; //simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE @@ -228,7 +231,7 @@ ; //small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt -small_stmt : expr_stmt -> ^(Expr expr_stmt) +small_stmt : expr_stmt | print_stmt | del_stmt | pass_stmt @@ -406,7 +409,7 @@ ; //suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT -suite : simple_stmt -> ^(Stmt simple_stmt) +suite : simple_stmt | NEWLINE! INDENT (stmt)+ DEDENT ; @@ -488,26 +491,37 @@ ; //power: atom trailer* ['**' factor] -power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR^ factor)? +power : (atom LPAREN) => atom (trailer)* powers? -> ^(Call atom (trailer)*) + | atom (trailer)* powers? ; +powers : (options {greedy=true;}:DOUBLESTAR^ factor) + ; + //atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ -atom : LPAREN (testlist)? RPAREN -> ^(Parens testlist?) - | LBRACK (listmaker)? RBRACK -> ^(List listmaker?) +atom : LPAREN + ( testlist -> testlist + | -> Tuple + ) + RPAREN + | LBRACK + ( listmaker -> listmaker + | -> List + ) RBRACK | LCURLY (dictmaker)? RCURLY -> ^(Dict dictmaker?) | BACKQUOTE testlist BACKQUOTE -> ^(Repr testlist) - | NAME - | INT - | LONGINT - | FLOAT - | COMPLEX - | (STRING)+ -> ^(String STRING+) + | NAME -> ^(Name NAME) + | INT -> ^(Num INT) + | LONGINT -> ^(Num LONGINT) + | FLOAT -> ^(Num FLOAT) + | COMPLEX -> ^(Num COMPLEX) + | (STRING)+ -> ^(Str STRING+) ; //listmaker: test ( list_for | (',' test)* [','] ) listmaker : test - ( list_for -> ^(ListComp list_for) - | (options {greedy=true;}:COMMA test)* -> test+ + ( list_for -> ^(ListComp test list_for) + | (options {greedy=true;}:COMMA test)* -> ^(List test+) ) (COMMA)? ; @@ -517,7 +531,7 @@ ; //trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -trailer : LPAREN (arglist)? RPAREN -> ^(ArgList arglist?) +trailer : LPAREN! (arglist)? RPAREN! | LBRACK subscriptlist RBRACK -> ^(SubscriptList subscriptlist) | DOT NAME ; @@ -543,8 +557,8 @@ ; //testlist: test (',' test)* [','] -testlist : test (options {k=2;}: COMMA test)* (COMMA)? - -> test+ +testlist : (test COMMA) => test (options {k=2;}: COMMA test)* (COMMA)? -> ^(Tuple test+) + | test -> test ; //XXX: @@ -568,7 +582,7 @@ | DOUBLESTAR kwargs=test )? )? - -> ^(Arguments argument+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? + -> ^(Args argument+) ^(StarArgs $starargs)? ^(KWArgs $kwargs)? | STAR starargs=test (COMMA DOUBLESTAR kwargs=test)? -> ^(StarArgs $starargs) ^(KWArgs $kwargs)? | DOUBLESTAR kwargs=test @@ -587,10 +601,12 @@ //list_for: 'for' exprlist 'in' testlist_safe [list_iter] list_for : 'for' exprlist 'in' testlist (list_iter)? + -> ^(ListCompFor exprlist ^(Iter testlist list_iter?)) ; //list_if: 'if' test [list_iter] list_if : 'if' test (list_iter)? + -> ^(ListCompIf test list_iter?) ; //XXX: Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-27 22:33:15 UTC (rev 3737) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-28 00:30:51 UTC (rev 3738) @@ -10,8 +10,8 @@ } module - : stmt* { - //System.out.println("<STMT>"); + : ^(Module ^(Body stmt*)) { + System.out.println("<STMT>"); } ; @@ -35,11 +35,7 @@ } ; -stmt: - ^(Stmt any_stmt+) - ; - -any_stmt //combines simple_stmt and compound_stmt from Python.g +stmt //combines simple_stmt and compound_stmt from Python.g : expr_stmt | print_stmt | del_stmt @@ -58,11 +54,11 @@ ; expr_stmt - : ^(Expr testlist) { + : test { } - | ^(Expr ^(augassign targ=test value=test)) { + | ^(augassign targ=test value=test) { } - | ^(Expr ^(Assign (^(Target testlist))+ ^(Value testlist))) { + | ^(Assign (^(Target test))+ ^(Value test)) { } ; @@ -82,7 +78,7 @@ ; print_stmt - : ^(Print RIGHTSHIFT? testlist?) + : ^(Print RIGHTSHIFT? test?) { } ; @@ -107,10 +103,10 @@ continue_stmt : 'continue' ; -return_stmt : ^(Return (testlist)?) +return_stmt : ^(Return (test)?) ; -yield_stmt : ^(Yield testlist) +yield_stmt : ^(Yield test) ; raise_stmt: ^(Raise (^(Type test))? (^(Inst test))? (^(Tback test))?) @@ -153,7 +149,7 @@ while_stmt : ^(While test ^(Body suite) (^(Else suite))?) ; -for_stmt : ^(For exprlist ^(In testlist) ^(Body suite) (^(Else suite))?) +for_stmt : ^(For exprlist ^(In test) ^(Body suite) (^(Else suite))?) ; try_stmt : ^(TryExcept ^(Body suite) except_clause+ (^(Else suite))?) @@ -209,22 +205,23 @@ //FIXME: lots of placeholders atom - : ^(List testlist?) {} + : ^(List test*) {} | ^(ListComp list_for) {} - | ^(Parens testlist?) {} - | ^(Dict testlist?) {} - | ^(Repr testlist?) {} - | NAME {} - | INT {} - | LONGINT {} - | FLOAT {} - | COMPLEX {} + | ^(Tuple test*) {} + | ^(Dict test*) {} + | ^(Repr test*) {} + | ^(Name NAME) {} +// | ^(Num (INT|LONGINT|FLOAT|COMPLEX)) {} + | ^(Num INT) {} + | ^(Num LONGINT) {} + | ^(Num FLOAT) {} + | ^(Num COMPLEX) {} | stringlist { } ; stringlist - : ^(String string+) {} + : ^(Str string+) {} ; string @@ -256,7 +253,7 @@ ; classdef - : ^(ClassDef ^(Name NAME) (^(Bases testlist))? ^(Body suite)) { + : ^(ClassDef ^(Name NAME) (^(Bases test*))? ^(Body suite)) { //System.out.println("class "); } ; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2007-11-28 03:34:29
|
Revision: 3740 http://jython.svn.sourceforge.net/jython/?rev=3740&view=rev Author: fwierzbicki Date: 2007-11-27 19:34:05 -0800 (Tue, 27 Nov 2007) Log Message: ----------- update of PythonWalker.g to better fit with previous changes in Python.g Modified Paths: -------------- trunk/sandbox/wierzbicki/antlr/grammar/Python.g trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g Modified: trunk/sandbox/wierzbicki/antlr/grammar/Python.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-28 02:26:10 UTC (rev 3739) +++ trunk/sandbox/wierzbicki/antlr/grammar/Python.g 2007-11-28 03:34:05 UTC (rev 3740) @@ -127,7 +127,6 @@ Lambda; Repr; BinOp; - ArgList; Subscript; SubscriptList; Target; @@ -148,6 +147,7 @@ ListCompFor; ListCompIf; Iter; + GetAttr; } @header { @@ -491,13 +491,9 @@ ; //power: atom trailer* ['**' factor] -power : (atom LPAREN) => atom (trailer)* powers? -> ^(Call atom (trailer)*) - | atom (trailer)* powers? +power : atom (trailer)* (options {greedy=true;}:DOUBLESTAR^ factor)? ; -powers : (options {greedy=true;}:DOUBLESTAR^ factor) - ; - //atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ atom : LPAREN ( testlist -> testlist @@ -531,9 +527,9 @@ ; //trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -trailer : LPAREN! (arglist)? RPAREN! +trailer : LPAREN (arglist)? RPAREN -> ^(Call arglist?) | LBRACK subscriptlist RBRACK -> ^(SubscriptList subscriptlist) - | DOT NAME + | DOT NAME -> ^(GetAttr NAME) ; //subscriptlist: subscript (',' subscript)* [','] Modified: trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g =================================================================== --- trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-28 02:26:10 UTC (rev 3739) +++ trunk/sandbox/wierzbicki/antlr/grammar/PythonWalker.g 2007-11-28 03:34:05 UTC (rev 3740) @@ -11,11 +11,10 @@ module : ^(Module ^(Body stmt*)) { - System.out.println("<STMT>"); + //System.out.println("<STMT>"); } ; - funcdef : ^(FunctionDef ^(Name NAME) ^(Args varargslist?) ^(Body suite)) { } @@ -31,7 +30,7 @@ ; defparameter - : NAME (ASSIGN test )? { + : ^(Name NAME) (ASSIGN test )? { } ; @@ -206,12 +205,13 @@ //FIXME: lots of placeholders atom : ^(List test*) {} - | ^(ListComp list_for) {} + | ^(ListComp test list_for) {} | ^(Tuple test*) {} | ^(Dict test*) {} | ^(Repr test*) {} - | ^(Name NAME) {} -// | ^(Num (INT|LONGINT|FLOAT|COMPLEX)) {} + | ^(Name NAME) { + //System.out.println("matched NAME:" + $NAME.text); + } | ^(Num INT) {} | ^(Num LONGINT) {} | ^(Num FLOAT) {} @@ -232,9 +232,9 @@ ; trailer - : ^(ArgList arglist?) + : ^(Call arglist?) | ^(SubscriptList subscriptlist) - | DOT NAME + | ^(GetAttr NAME) ; subscriptlist @@ -259,7 +259,7 @@ ; arglist - : ^(Arguments argument+) (^(StarArgs test))? (^(KWArgs test))? + : ^(Args argument+) (^(StarArgs test))? (^(KWArgs test))? | ^(StarArgs test) (^(KWArgs test))? | ^(KWArgs test) ; @@ -271,9 +271,10 @@ | list_if ; -list_for: 'for' exprlist 'in' testlist (list_iter)? - ; +list_for : ^(ListCompFor exprlist ^(Iter testlist list_iter?)) + ; -list_if: 'if' test (list_iter)? - ; +list_if : ^(ListCompIf test list_iter?) + ; + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |