From: <fwi...@us...> - 2008-07-17 16:38:26
|
Revision: 4964 http://jython.svn.sourceforge.net/jython/?rev=4964&view=rev Author: fwierzbicki Date: 2008-07-17 16:38:20 +0000 (Thu, 17 Jul 2008) Log Message: ----------- start on lambda, comp_op, and list comp. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-07-17 04:59:53 UTC (rev 4963) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-07-17 16:38:20 UTC (rev 4964) @@ -77,7 +77,7 @@ tokens { INDENT; DEDENT; - + ModuleTok; Interactive; Expression; @@ -165,6 +165,9 @@ import org.python.antlr.ast.Break; import org.python.antlr.ast.Call; import org.python.antlr.ast.ClassDef; +import org.python.antlr.ast.cmpopType; +import org.python.antlr.ast.Compare; +import org.python.antlr.ast.comprehensionType; import org.python.antlr.ast.Context; import org.python.antlr.ast.Continue; import org.python.antlr.ast.Delete; @@ -182,6 +185,7 @@ import org.python.antlr.ast.ImportFrom; import org.python.antlr.ast.Index; import org.python.antlr.ast.keywordType; +import org.python.antlr.ast.Lambda; import org.python.antlr.ast.modType; import org.python.antlr.ast.Module; import org.python.antlr.ast.Name; @@ -990,7 +994,7 @@ @after { $exec_stmt.tree = $stype; } - : keyEXEC expr[expr_contextType.Load] ('in' t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? { + : keyEXEC expr[expr_contextType.Load] (IN t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? { $stype = new Exec($expr.start, (exprType)$expr.tree, (exprType)$t1.tree, (exprType)$t2.tree); } ; @@ -1111,7 +1115,23 @@ ; //comparison: expr (comp_op expr)* -comparison[expr_contextType ctype]: expr[ctype] (comp_op^ expr[ctype])* +comparison[expr_contextType ctype] +@after { + if ($comparison.tree instanceof Compare) { + System.out.println("found Compare"); + Compare c = (Compare)$comparison.tree; + //FIXME: Only handling the case of a single comparison. + c.left = (exprType)c.getChild(0); + System.out.println("c.left:" + c.left); + c.comparators = new exprType[]{(exprType)c.getChild(1)}; + switch (c.getType()) { + case IN: + c.ops = new cmpopType[]{cmpopType.In}; + break; + } + } +} + : expr[ctype] (IN<Compare>^ expr[ctype])* ; //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' @@ -1122,8 +1142,8 @@ | LESSEQUAL | ALT_NOTEQUAL | NOTEQUAL - | 'in' - | NOT 'in' -> NotIn + | IN + | NOT IN -> NotIn | 'is' | 'is' NOT -> IsNot ; @@ -1265,7 +1285,7 @@ //lambdef: 'lambda' [varargslist] ':' test lambdef: LAMBDA (varargslist)? COLON test[expr_contextType.Load] {debug("parsed lambda");} - -> ^(LAMBDA varargslist? ^(Body test)) + -> ^(LAMBDA<Lambda>[$LAMBDA, $varargslist.args, (exprType)$test.tree]) ; //trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME @@ -1414,7 +1434,7 @@ //list_for: 'for' exprlist 'in' testlist_safe [list_iter] list_for : FOR exprlist[expr_contextType.Load] IN testlist[expr_contextType.Load] (list_iter)? - -> ^(ListFor ^(Target exprlist) ^(IN testlist) ^(Ifs list_iter)?) + -> ^(FOR<comprehensionType>[$FOR, $exprlist.etype, $testlist.etype, null]) ; //list_if: 'if' test [list_iter] This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-07 11:56:34
|
Revision: 5101 http://jython.svn.sourceforge.net/jython/?rev=5101&view=rev Author: fwierzbicki Date: 2008-08-07 11:56:31 +0000 (Thu, 07 Aug 2008) Log Message: ----------- merge unified grammar with trunk. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-07 02:27:08 UTC (rev 5100) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-07 11:56:31 UTC (rev 5101) @@ -214,15 +214,15 @@ } @members { - //If you want to use antlr's default error recovery mechanisms change this - //and the same one in the lexer to true. - public boolean antlrErrorHandling = false; + boolean debugOn = false; - //XXX: only used for single_input -- seems kludgy. - public boolean inSingle = false; + private ErrorHandler errorHandler; + private boolean seenSingleOuterSuite = false; - boolean debugOn = false; + public void setErrorHandler(ErrorHandler eh) { + this.errorHandler = eh; + } private void debug(String message) { if (debugOn) { @@ -409,8 +409,8 @@ } private exprType makeAssignValue(List rhs) { - testlist_return r = (testlist_return)rhs.get(rhs.size() -1); - exprType value = (exprType)r.getTree(); + exprType value = (exprType)rhs.get(rhs.size() -1); + if (value instanceof Context) { //XXX: for Tuples, etc -- will need to recursively set to expr_contextType.Load. ((Context)value).setContext(expr_contextType.Load); @@ -453,12 +453,15 @@ keywordType[] makeKeywords(List args) { List<keywordType> k = new ArrayList<keywordType>(); - for(int i=0;i<args.size();i++) { - exprType[] e = (exprType[])args.get(i); - Name arg = (Name)e[0]; - k.add(new keywordType(arg, arg.id, e[1])); + if (args != null) { + for(int i=0;i<args.size();i++) { + exprType[] e = (exprType[])args.get(i); + Name arg = (Name)e[0]; + k.add(new keywordType(arg, arg.id, e[1])); + } + return k.toArray(new keywordType[k.size()]); } - return k.toArray(new keywordType[k.size()]); + return new keywordType[0]; } Object makeFloat(Token t) { @@ -574,23 +577,22 @@ } Token extractStringToken(List s) { - //XXX: really we want the *last* one. - return (Token)s.get(0); + return (Token)s.get(s.size() - 1); } protected void mismatch(IntStream input, int ttype, BitSet follow) throws RecognitionException { - if (antlrErrorHandling) { + if (errorHandler.isRecoverable()) { super.mismatch(input, ttype, follow); } else { throw new MismatchedTokenException(ttype, input); } } - protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow) - throws RecognitionException - { - if (antlrErrorHandling) { + protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow) + throws RecognitionException + { + if (errorHandler.isRecoverable()) { return super.recoverFromMismatchedToken(input, ttype, follow); } mismatch(input, ttype, follow); @@ -601,13 +603,9 @@ @rulecatch { catch (RecognitionException re) { - if (antlrErrorHandling) { - reportError(re); - recover(input,re); - retval.tree = (PythonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); - } else { - throw new ParseException(re); - } + errorHandler.reportError(this, re); + errorHandler.recover(this, input,re); + retval.tree = (PythonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); } } @@ -622,19 +620,23 @@ * 4] */ -//If you want to use antlr's default error recovery mechanisms change this -//and the same one in the parser to true. -public boolean antlrErrorHandling = false; +//If you want to use another error recovery mechanisms change this +//and the same one in the parser. +private ErrorHandler errorHandler; -//XXX: Hopefully we can remove inSingle when we get PyCF_DONT_IMPLY_DEDENT support. -public boolean inSingle = false; int implicitLineJoiningLevel = 0; int startPos=-1; + public void setErrorHandler(ErrorHandler eh) { + this.errorHandler = eh; + } + + /** + * Taken directly from antlr's Lexer.java -- needs to be re-integrated every time + * we upgrade from Antlr (need to consider a Lexer subclass, though the issue would + * remain). + */ public Token nextToken() { - if (antlrErrorHandling) { - return super.nextToken(); - } while (true) { state.token = null; state.channel = Token.DEFAULT_CHANNEL; @@ -654,16 +656,19 @@ continue; } return state.token; + } catch (NoViableAltException nva) { + errorHandler.reportError(this, nva); + errorHandler.recover(this, nva); // throw out current char and try again + } catch (RecognitionException re) { + errorHandler.reportError(this, re); + // match() routine has already called recover() } - catch (RecognitionException re) { - throw new ParseException(re); - } } } } //single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -single_input : NEWLINE -> ^(Interactive) +single_input : NEWLINE? -> ^(Interactive) | simple_stmt -> ^(Interactive simple_stmt) | compound_stmt NEWLINE -> ^(Interactive compound_stmt) ; @@ -686,6 +691,44 @@ ) ; +//attr is here for Java compatibility. A Java foo.getIf() can be called from Jython as foo.if +// so we need to support any keyword as an attribute. + +attr + : NAME + | AND + | AS + | ASSERT + | BREAK + | CLASS + | CONTINUE + | DEF + | DELETE + | ELIF + | EXCEPT + | EXEC + | FINALLY + | FROM + | FOR + | GLOBAL + | IF + | IMPORT + | IN + | IS + | LAMBDA + | NOT + | OR + | ORELSE + | PASS + | PRINT + | RAISE + | RETURN + | TRY + | WHILE + | WITH + | YIELD + ; + //decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE decorator returns [exprType etype] @after { @@ -953,22 +996,19 @@ @after { $import_as_name.tree = $atype; } - : name=NAME (keyAS asname=NAME)? { + : name=NAME (AS asname=NAME)? { $atype = new aliasType($name, $name.text, $asname.text); } ; -//XXX: when does CPython Grammar match "dotted_name NAME NAME"? This may be a big -// problem because of the keyAS rule, which matches NAME (needed to allow -// 'as' to be a method name for Java integration). - +//XXX: when does CPython Grammar match "dotted_name NAME NAME"? //dotted_as_name: dotted_name [('as' | NAME) NAME] dotted_as_name returns [aliasType atype] @after { $dotted_as_name.tree = $atype; } - : dotted_name (keyAS NAME)? { + : dotted_name (AS NAME)? { $atype = new aliasType($NAME, $dotted_name.text, $NAME.text); } ; @@ -981,7 +1021,7 @@ ; //dotted_name: NAME ('.' NAME)* -dotted_name : NAME (DOT NAME)* +dotted_name : NAME (DOT attr)* ; //global_stmt: 'global' NAME (',' NAME)* @@ -994,7 +1034,7 @@ @after { $exec_stmt.tree = $stype; } - : keyEXEC expr[expr_contextType.Load] (IN t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? { + : EXEC expr[expr_contextType.Load] (IN t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? { $stype = new Exec($expr.start, (exprType)$expr.tree, (exprType)$t1.tree, (exprType)$t2.tree); } ; @@ -1075,7 +1115,7 @@ //with_var: ('as' | NAME) expr with_var returns [exprType etype] - : (keyAS | NAME) expr[expr_contextType.Load] { + : (AS | NAME) expr[expr_contextType.Load] { $etype = (exprType)$expr.tree; } ; @@ -1144,8 +1184,8 @@ | NOTEQUAL | IN | NOT IN -> NotIn - | 'is' - | 'is' NOT -> IsNot + | IS + | IS NOT -> IsNot ; //expr: xor_expr ('|' xor_expr)* @@ -1254,7 +1294,7 @@ | LONGINT -> ^(NumTok<Num>[$LONGINT, makeInt($LONGINT)]) | FLOAT -> ^(NumTok<Num>[$FLOAT, makeFloat($FLOAT)]) | COMPLEX -> ^(NumTok<Num>[$COMPLEX, makeComplex($COMPLEX)]) - | (S+=STRING)+ {debug("S+: " + $S);} + | (S+=STRING)+ -> ^(StrTok<Str>[extractStringToken($S), extractStrings($S)]) ; @@ -1294,7 +1334,7 @@ ) RPAREN | LBRACK s=subscriptlist RBRACK -> $s - | DOT^ NAME {debug("motched DOT^ NAME");} + | DOT^ attr {debug("motched DOT^ NAME");} ; //subscriptlist: subscript (',' subscript)* [','] @@ -1465,56 +1505,34 @@ //XXX: //testlist1: test (',' test)* -//These are all Python keywords that are not Java keywords -//This means that Jython needs to support these as NAMEs -//unlike CPython. For now I have only done this for 'as' -//and 'exec'. - -//keyAND : {input.LT(1).getText().equals("and")}? NAME ; -keyAS : {input.LT(1).getText().equals("as")}? NAME ; -//keyDEF : {input.LT(1).getText().equals("def")}? NAME ; -//keyDEL : {input.LT(1).getText().equals("del")}? NAME ; -//keyELIF : {input.LT(1).getText().equals("elif")}? NAME ; -//keyEXCEPT : {input.LT(1).getText().equals("except")}? NAME ; -keyEXEC : {input.LT(1).getText().equals("exec")}? NAME ; -//keyFROM : {input.LT(1).getText().equals("from")}? NAME ; -//keyGLOBAL : {input.LT(1).getText().equals("global")}? NAME ; -//keyIN : {input.LT(1).getText().equals("in")}? NAME ; -//keyIS : {input.LT(1).getText().equals("is")}? NAME ; -//keyLAMBDA : {input.LT(1).getText().equals("lambda")}? NAME ; -//keyNOT : {input.LT(1).getText().equals("not")}? NAME ; -//keyOR : {input.LT(1).getText().equals("or")}? NAME ; -//keyPASS : {input.LT(1).getText().equals("pass")}? NAME ; -//keyPRINT : {input.LT(1).getText().equals("print")}? NAME ; -//keyRAISE : {input.LT(1).getText().equals("raise")}? NAME ; -//keyWITH : {input.LT(1).getText().equals("with")}? NAME ; -//keyYIELD : {input.LT(1).getText().equals("yield")}? NAME ; - -DEF : 'def' ; +AS : 'as' ; +ASSERT : 'assert' ; +BREAK : 'break' ; CLASS : 'class' ; -PRINT : 'print' ; -BREAK : 'break' ; CONTINUE : 'continue' ; -RETURN : 'return' ; -RAISE : 'raise' ; -PASS : 'pass' ; -IMPORT : 'import' ; +DEF : 'def' ; +DELETE : 'del' ; +ELIF : 'elif' ; +EXCEPT : 'except' ; +EXEC : 'exec' ; +FINALLY : 'finally' ; FROM : 'from' ; FOR : 'for' ; +GLOBAL : 'global' ; +IF : 'if' ; +IMPORT : 'import' ; +IN : 'in' ; +IS : 'is' ; +LAMBDA : 'lambda' ; ORELSE : 'else' ; -ELIF : 'elif' ; -IN : 'in' ; -IF : 'if' ; +PASS : 'pass' ; +PRINT : 'print' ; +RAISE : 'raise' ; +RETURN : 'return' ; +TRY : 'try' ; WHILE : 'while' ; WITH : 'with' ; -LAMBDA : 'lambda' ; -GLOBAL : 'global' ; YIELD : 'yield' ; -ASSERT : 'assert' ; -FINALLY : 'finally' ; -DELETE : 'del' ; -EXCEPT : 'except' ; -TRY : 'try' ; LPAREN : '(' {implicitLineJoiningLevel++;} ; @@ -1655,7 +1673,12 @@ | '"""' (options {greedy=false;}:TRIQUOTE)* '"""' | '"' (ESC|~('\\'|'\n'|'"'))* '"' | '\'' (ESC|~('\\'|'\n'|'\''))* '\'' - ) + ) { + if (state.tokenStartLine != input.getLine()) { + state.tokenStartLine = input.getLine(); + state.tokenStartCharPositionInLine = -2; + } + } ; /** the two '"'? cause a warning -- is there a way to avoid that? */ @@ -1681,7 +1704,7 @@ */ CONTINUED_LINE : '\\' ('\r')? '\n' (' '|'\t')* { $channel=HIDDEN; } - ( nl=NEWLINE {emit(new ClassicToken(NEWLINE,nl.getText()));} + ( nl=NEWLINE {emit(new CommonToken(NEWLINE,nl.getText()));} | ) ; @@ -1693,12 +1716,11 @@ * Frank Wierzbicki added: Also ignore FORMFEEDS (\u000C). */ NEWLINE - : {inSingle}? => (('\u000C')?('\r')? '\n' ) - {if (implicitLineJoiningLevel>0 ) - $channel=HIDDEN; - } - | (('\u000C')?('\r')? '\n' )+ - {if ( startPos==0 || implicitLineJoiningLevel>0 ) +@init { + int newlines = 0; +} + : (('\u000C')?('\r')? '\n' {newlines++; } )+ { + if ( startPos==0 || implicitLineJoiningLevel>0 ) $channel=HIDDEN; } ; @@ -1715,24 +1737,42 @@ LEADING_WS @init { int spaces = 0; + int newlines = 0; } : {startPos==0}?=> ( {implicitLineJoiningLevel>0}? ( ' ' | '\t' )+ {$channel=HIDDEN;} - | ( ' ' { spaces++; } - | '\t' { spaces += 8; spaces -= (spaces \% 8); } - )+ - { - // make a string of n spaces where n is column number - 1 - char[] indentation = new char[spaces]; - for (int i=0; i<spaces; i++) { - indentation[i] = ' '; - } - String s = new String(indentation); - emit(new ClassicToken(LEADING_WS,new String(indentation))); - } - // kill trailing newline if present and then ignore - ( ('\r')? '\n' {if (state.token!=null) state.token.setChannel(HIDDEN); else $channel=HIDDEN;})* - // {state.token.setChannel(99); } + | ( ' ' { spaces++; } + | '\t' { spaces += 8; spaces -= (spaces \% 8); } + )+ + ( ('\r')? '\n' {newlines++; } + )* { + if (input.LA(1) != -1) { + // make a string of n spaces where n is column number - 1 + char[] indentation = new char[spaces]; + for (int i=0; i<spaces; i++) { + indentation[i] = ' '; + } + CommonToken c = new CommonToken(LEADING_WS,new String(indentation)); + c.setLine(input.getLine()); + c.setCharPositionInLine(input.getCharPositionInLine()); + emit(c); + // kill trailing newline if present and then ignore + if (newlines != 0) { + if (state.token!=null) { + state.token.setChannel(HIDDEN); + } else { + $channel=HIDDEN; + } + } + } else { + // make a string of n newlines + char[] nls = new char[newlines]; + for (int i=0; i<newlines; i++) { + nls[i] = '\n'; + } + emit(new CommonToken(NEWLINE,new String(nls))); + } + } ) ; @@ -1758,6 +1798,6 @@ $channel=HIDDEN; } : {startPos==0}?=> (' '|'\t')* '#' (~'\n')* '\n'+ - | {startPos>0}?=> '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#' + | '#' (~'\n')* // let NEWLINE handle \n unless char pos==0 for '#' ; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-09 12:36:42
|
Revision: 5118 http://jython.svn.sourceforge.net/jython/?rev=5118&view=rev Author: fwierzbicki Date: 2008-08-09 12:36:39 +0000 (Sat, 09 Aug 2008) Log Message: ----------- merge and add more compops. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-09 06:18:29 UTC (rev 5117) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-09 12:36:39 UTC (rev 5118) @@ -220,8 +220,11 @@ private boolean seenSingleOuterSuite = false; + private GrammarActions actions = new GrammarActions(); + public void setErrorHandler(ErrorHandler eh) { this.errorHandler = eh; + actions.setErrorHandler(eh); } private void debug(String message) { @@ -230,357 +233,6 @@ } } - private String makeFromText(List dots, String name) { - StringBuffer d = new StringBuffer(); - if (dots != null) { - for (int i=0;i<dots.size();i++) { - d.append("."); - } - } - if (name != null) { - d.append(name); - } - return d.toString(); - } - - private int makeLevel(List lev) { - if (lev == null) { - return 0; - } - return lev.size(); - } - - private aliasType[] makeStarAlias(Token t) { - return new aliasType[]{new aliasType(t, "*", null)}; - } - - private aliasType[] makeAliases(aliasType[] atypes) { - if (atypes == null) { - return new aliasType[0]; - } - return atypes; - } - - private exprType[] makeBases(exprType etype) { - if (etype != null) { - if (etype instanceof Tuple) { - return ((Tuple)etype).elts; - } - return new exprType[]{etype}; - } - return new exprType[0]; - } - - private String[] makeNames(List names) { - List<String> s = new ArrayList<String>(); - for(int i=0;i<names.size();i++) { - s.add(((Token)names.get(i)).getText()); - } - return s.toArray(new String[s.size()]); - } - - private void throwGenExpNotSoleArg(PythonTree t) { - throw new ParseException("Generator expression must be parenthesized if not sole argument", t); - } - - private exprType[] makeExprs(List exprs) { - return makeExprs(exprs, 0); - } - - private exprType[] makeExprs(List exprs, int start) { - if (exprs != null) { - List<exprType> result = new ArrayList<exprType>(); - for (int i=start; i<exprs.size(); i++) { - Object o = exprs.get(i); - if (o instanceof exprType) { - result.add((exprType)o); - } else if (o instanceof test_return) { - result.add((exprType)((test_return)o).tree); - } - } - return result.toArray(new exprType[result.size()]); - } - return new exprType[0]; - } - - private stmtType[] makeElses(List elseSuite, List elifs) { - stmtType[] o; - o = makeStmts(elseSuite); - if (elifs != null) { - ListIterator iter = elifs.listIterator(elifs.size()); - while (iter.hasPrevious()) { - If elif = (If)iter.previous(); - elif.orelse = o; - o = new stmtType[]{elif}; - } - } - return o; - } - - private stmtType[] makeStmts(List stmts) { - if (stmts != null) { - List<stmtType> result = new ArrayList<stmtType>(); - for (int i=0; i<stmts.size(); i++) { - Object o = stmts.get(i); - if (o instanceof stmtType) { - result.add((stmtType)o); - } else { - result.add((stmtType)((stmt_return)o).tree); - } - } - return (stmtType[])result.toArray(new stmtType[result.size()]); - } - return new stmtType[0]; - } - - private exprType makeDottedAttr(Token nameToken, List attrs) { - exprType current = new Name(nameToken, nameToken.getText(), expr_contextType.Load); - for (int i=attrs.size() - 1; i > -1; i--) { - Token t = ((PythonTree)attrs.get(i)).token; - current = new Attribute(t, current, t.getText(), - expr_contextType.Load); - } - return current; - } - - private While makeWhile(Token t, exprType test, List body, List orelse) { - stmtType[] o = makeStmts(orelse); - stmtType[] b = makeStmts(body); - return new While(t, test, b, o); - } - - private For makeFor(Token t, exprType target, exprType iter, List body, List orelse) { - stmtType[] o = makeStmts(orelse); - stmtType[] b = makeStmts(body); - return new For(t, target, iter, b, o); - } - - private stmtType makeTryExcept(Token t, List body, List handlers, List orelse, List finBody) { - stmtType[] b = (stmtType[])body.toArray(new stmtType[body.size()]); - excepthandlerType[] e = (excepthandlerType[])handlers.toArray(new excepthandlerType[handlers.size()]); - stmtType[] o; - if (orelse != null) { - o = (stmtType[])orelse.toArray(new stmtType[orelse.size()]); - } else { - o = new stmtType[0]; - } - - stmtType te = new TryExcept(t, b, e, o); - if (finBody == null) { - return te; - } - stmtType[] f = (stmtType[])finBody.toArray(new stmtType[finBody.size()]); - stmtType[] mainBody = new stmtType[]{te}; - return new TryFinally(t, mainBody, f); - } - - private TryFinally makeTryFinally(Token t, List body, List finBody) { - stmtType[] b = (stmtType[])body.toArray(new stmtType[body.size()]); - stmtType[] f = (stmtType[])finBody.toArray(new stmtType[finBody.size()]); - return new TryFinally(t, b, f); - } - - private FunctionDef makeFunctionDef(PythonTree t, PythonTree nameToken, argumentsType args, List funcStatements, List decorators) { - argumentsType a; - debug("Matched FunctionDef"); - if (args != null) { - a = args; - } else { - a = new argumentsType(t, new exprType[0], null, null, new exprType[0]); - } - stmtType[] s = (stmtType[])funcStatements.toArray(new stmtType[funcStatements.size()]); - exprType[] d; - if (decorators != null) { - d = (exprType[])decorators.toArray(new exprType[decorators.size()]); - } else { - d = new exprType[0]; - } - return new FunctionDef(t, nameToken.getText(), a, s, d); - } - - private exprType[] makeAssignTargets(exprType lhs, List rhs) { - exprType[] e = new exprType[rhs.size()]; - e[0] = lhs; - for(int i=0;i<rhs.size() - 1;i++) { - testlist_return r = (testlist_return)rhs.get(i); - e[i] = (exprType)r.getTree(); - } - return e; - } - - private exprType makeAssignValue(List rhs) { - exprType value = (exprType)rhs.get(rhs.size() -1); - - if (value instanceof Context) { - //XXX: for Tuples, etc -- will need to recursively set to expr_contextType.Load. - ((Context)value).setContext(expr_contextType.Load); - } - return value; - } - - private argumentsType makeArgumentsType(Token t, List params, Token snameToken, - Token knameToken, List defaults) { - debug("Matched Arguments"); - - exprType[] p; - if (params == null) { - p = new exprType[0]; - } else { - p = (exprType[])params.toArray(new exprType[params.size()]); - } - exprType[] d = (exprType[])defaults.toArray(new exprType[defaults.size()]); - String s; - String k; - if (snameToken == null) { - s = null; - } else { - s = snameToken.getText(); - } - if (knameToken == null) { - k = null; - } else { - k = knameToken.getText(); - } - return new argumentsType(t, p, s, k, d); - } - - exprType[] extractArgs(List args) { - if (args == null) { - return new exprType[0]; - } - return (exprType[])args.toArray(new exprType[args.size()]); - } - - keywordType[] makeKeywords(List args) { - List<keywordType> k = new ArrayList<keywordType>(); - if (args != null) { - for(int i=0;i<args.size();i++) { - exprType[] e = (exprType[])args.get(i); - Name arg = (Name)e[0]; - k.add(new keywordType(arg, arg.id, e[1])); - } - return k.toArray(new keywordType[k.size()]); - } - return new keywordType[0]; - } - - Object makeFloat(Token t) { - debug("makeFloat matched " + t.getText()); - return Py.newFloat(Double.valueOf(t.getText())); - } - - Object makeComplex(Token t) { - String s = t.getText(); - s = s.substring(0, s.length() - 1); - return Py.newImaginary(Double.valueOf(s)); - } - - Object makeInt(Token t) { - debug("Num matched " + t.getText()); - String s = t.getText(); - int radix = 10; - if (s.startsWith("0x") || s.startsWith("0X")) { - radix = 16; - s = s.substring(2, s.length()); - } else if (s.startsWith("0")) { - radix = 8; - } - if (s.endsWith("L") || s.endsWith("l")) { - s = s.substring(0, s.length()-1); - return Py.newLong(new BigInteger(s, radix)); - } - int ndigits = s.length(); - int i=0; - while (i < ndigits && s.charAt(i) == '0') - i++; - if ((ndigits - i) > 11) { - return Py.newLong(new BigInteger(s, radix)); - } - - long l = Long.valueOf(s, radix).longValue(); - if (l > 0xffffffffl || (l > Integer.MAX_VALUE)) { - return Py.newLong(new BigInteger(s, radix)); - } - return Py.newInteger((int) l); - } - - class StringPair { - private String s; - private boolean unicode; - - StringPair(String s, boolean unicode) { - this.s = s; - this.unicode = unicode; - } - String getString() { - return s; - } - - boolean isUnicode() { - return unicode; - } - } - - PyString extractStrings(List s) { - boolean ustring = false; - Token last = null; - StringBuffer sb = new StringBuffer(); - Iterator iter = s.iterator(); - while (iter.hasNext()) { - last = (Token)iter.next(); - StringPair sp = extractString(last); - if (sp.isUnicode()) { - ustring = true; - } - sb.append(sp.getString()); - } - if (ustring) { - return new PyUnicode(sb.toString()); - } - return new PyString(sb.toString()); - } - - StringPair extractString(Token t) { - String s = t.getText(); - char quoteChar = s.charAt(0); - int start=0; - boolean ustring = false; - if (quoteChar == 'u' || quoteChar == 'U') { - ustring = true; - start++; - } - quoteChar = s.charAt(start); - boolean raw = false; - if (quoteChar == 'r' || quoteChar == 'R') { - raw = true; - start++; - } - int quotes = 3; - if (s.length() - start == 2) { - quotes = 1; - } - if (s.charAt(start) != s.charAt(start+1)) { - quotes = 1; - } - - if (raw) { - return new StringPair(s.substring(quotes+start, s.length()-quotes), ustring); - } else { - StringBuffer sb = new StringBuffer(s.length()); - char[] ca = s.toCharArray(); - int n = ca.length-quotes; - int i=quotes+start; - int last_i=i; - return new StringPair(PyString.decode_UnicodeEscape(s, i, n, "strict", ustring), ustring); - //return decode_UnicodeEscape(s, i, n, "strict", ustring); - } - } - - Token extractStringToken(List s) { - return (Token)s.get(s.size() - 1); - } - - protected void mismatch(IntStream input, int ttype, BitSet follow) throws RecognitionException { if (errorHandler.isRecoverable()) { super.mismatch(input, ttype, follow); @@ -675,7 +327,7 @@ //file_input: (NEWLINE | stmt)* ENDMARKER file_input - : (NEWLINE | s+=stmt)+ -> ^(ModuleTok<Module>[$file_input.start, makeStmts($s)]) + : (NEWLINE | s+=stmt)+ -> ^(ModuleTok<Module>[$file_input.start, actions.makeStmts($s)]) | -> ^(ModuleTok<Module>[$file_input.start, new stmtType[0\]]) ; @@ -686,7 +338,7 @@ //not in CPython's Grammar file dotted_attr returns [exprType etype] : n1=NAME - ( (DOT n2+=dotted_attr)+ { $etype = makeDottedAttr($n1, $n2); } + ( (DOT n2+=dotted_attr)+ { $etype = actions.makeDottedAttr($n1, $n2); } | { $etype = new Name($NAME, $NAME.text, expr_contextType.Load); } ) ; @@ -737,8 +389,8 @@ : AT dotted_attr //XXX: ignoring the arglist and Call generation right now. ( LPAREN (arglist - {$etype = new Call($LPAREN, $dotted_attr.etype, makeExprs($arglist.args), - makeKeywords($arglist.keywords), $arglist.starargs, $arglist.kwargs);} + {$etype = new Call($LPAREN, $dotted_attr.etype, actions.makeExprs($arglist.args), + actions.makeKeywords($arglist.keywords), $arglist.starargs, $arglist.kwargs);} | {$etype = $dotted_attr.etype;} ) RPAREN @@ -754,8 +406,8 @@ //funcdef: [decorators] 'def' NAME parameters ':' suite funcdef : decorators? DEF NAME parameters COLON suite - -> ^(DEF<FunctionDef>[$DEF, $NAME.text, $parameters.args, makeStmts($suite.stmts), - makeExprs($decorators.etypes)]) + -> ^(DEF<FunctionDef>[$DEF, $NAME.text, $parameters.args, actions.makeStmts($suite.stmts), + actions.makeExprs($decorators.etypes)]) ; //parameters: '(' [varargslist] ')' @@ -797,11 +449,11 @@ | DOUBLESTAR kwargs=NAME )? )? - {$args = makeArgumentsType($varargslist.start, $d, $starargs, $kwargs, defaults);} + {$args = actions.makeArgumentsType($varargslist.start, $d, $starargs, $kwargs, defaults);} | STAR starargs=NAME (COMMA DOUBLESTAR kwargs=NAME)?{debug("parsed varargslist STARARGS");} - {$args = makeArgumentsType($varargslist.start, $d, $starargs, $kwargs, defaults);} + {$args = actions.makeArgumentsType($varargslist.start, $d, $starargs, $kwargs, defaults);} | DOUBLESTAR kwargs=NAME {debug("parsed varargslist KWS");} - {$args = makeArgumentsType($varargslist.start, $d, null, $kwargs, defaults);} + {$args = actions.makeArgumentsType($varargslist.start, $d, null, $kwargs, defaults);} ; //fpdef: NAME | '(' fplist ')' @@ -810,7 +462,7 @@ | (LPAREN fpdef[expr_contextType.Load] COMMA) => LPAREN fplist RPAREN -> fplist | LPAREN fplist RPAREN - -> ^(LPAREN<Tuple>[$fplist.start, makeExprs($fplist.etypes), expr_contextType.Store]) + -> ^(LPAREN<Tuple>[$fplist.start, actions.makeExprs($fplist.etypes), expr_contextType.Store]) ; //fplist: fpdef (',' fpdef)* [','] @@ -848,8 +500,8 @@ ) ( (augassign yield_expr -> ^(augassign $lhs yield_expr)) | (augassign rhs=testlist[expr_contextType.Load] -> ^(augassign $lhs $rhs)) - | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(AssignTok<Assign>[$at, makeAssignTargets((exprType)$lhs.tree, $t), makeAssignValue($t)])) - | ((ay=ASSIGN y+=yield_expr)+ -> ^(AssignTok<Assign>[$ay, makeAssignTargets((exprType)$lhs.tree, $y), makeAssignValue($y)])) + | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(AssignTok<Assign>[$at, actions.makeAssignTargets((exprType)$lhs.tree, $t), actions.makeAssignValue($t)])) + | ((ay=ASSIGN y+=yield_expr)+ -> ^(AssignTok<Assign>[$ay, actions.makeAssignTargets((exprType)$lhs.tree, $y), actions.makeAssignValue($y)])) | -> ExprTok<Expr>[$lhs.start, (exprType)$lhs.tree] ) ; @@ -874,9 +526,9 @@ // '>>' test [ (',' test)+ [','] ] ) print_stmt : PRINT ( t1=printlist - -> ^(PRINT<Print>[$PRINT, null, makeExprs($t1.elts), $t1.newline]) + -> ^(PRINT<Print>[$PRINT, null, actions.makeExprs($t1.elts), $t1.newline]) | RIGHTSHIFT t2=printlist2 - -> ^(PRINT<Print>[$PRINT, (exprType)$t2.elts.get(0), makeExprs($t2.elts, 1), $t2.newline]) + -> ^(PRINT<Print>[$PRINT, (exprType)$t2.elts.get(0), actions.makeExprs($t2.elts, 1), $t2.newline]) | -> ^(PRINT<Print>[$PRINT, null, new exprType[0\], false]) ) @@ -920,7 +572,7 @@ //del_stmt: 'del' exprlist del_stmt : DELETE exprlist2 - -> ^(DELETE<Delete>[$DELETE, makeExprs($exprlist2.etypes)]) + -> ^(DELETE<Delete>[$DELETE, actions.makeExprs($exprlist2.etypes)]) ; //pass_stmt: 'pass' @@ -976,11 +628,11 @@ // 'import' ('*' | '(' import_as_names ')' | import_as_names)) import_from: FROM (d+=DOT* dotted_name | d+=DOT+) IMPORT (STAR - -> ^(FROM<ImportFrom>[$FROM, makeFromText($d, $dotted_name.text), makeStarAlias($STAR), makeLevel($d)]) + -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeStarAlias($STAR), actions.makeLevel($d)]) | i1=import_as_names - -> ^(FROM<ImportFrom>[$FROM, makeFromText($d, $dotted_name.text), makeAliases($i1.atypes), makeLevel($d)]) + -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeAliases($i1.atypes), actions.makeLevel($d)]) | LPAREN i2=import_as_names RPAREN - -> ^(FROM<ImportFrom>[$FROM, makeFromText($d, $dotted_name.text), makeAliases($i2.atypes), makeLevel($d)]) + -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeAliases($i2.atypes), actions.makeLevel($d)]) ) ; @@ -1026,7 +678,7 @@ //global_stmt: 'global' NAME (',' NAME)* global_stmt : GLOBAL n+=NAME (COMMA n+=NAME)* - -> ^(GLOBAL<Global>[$GLOBAL, makeNames($n)]) + -> ^(GLOBAL<Global>[$GLOBAL, actions.makeNames($n)]) ; //exec_stmt: 'exec' expr ['in' test [',' test]] @@ -1056,12 +708,12 @@ //if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] if_stmt: IF test[expr_contextType.Load] COLON ifsuite=suite elifs+=elif_clause* (ORELSE COLON elsesuite=suite)? - -> ^(IF<If>[$IF, (exprType)$test.tree, makeStmts($ifsuite.stmts), makeElses($elsesuite.stmts, $elifs)]) + -> ^(IF<If>[$IF, (exprType)$test.tree, actions.makeStmts($ifsuite.stmts), actions.makeElses($elsesuite.stmts, $elifs)]) ; //not in CPython's Grammar file elif_clause : ELIF test[expr_contextType.Load] COLON suite - -> ^(ELIF<If>[$ELIF, (exprType)$test.tree, makeStmts($suite.stmts), new stmtType[0\]]) + -> ^(ELIF<If>[$ELIF, (exprType)$test.tree, actions.makeStmts($suite.stmts), new stmtType[0\]]) ; //while_stmt: 'while' test ':' suite ['else' ':' suite] @@ -1070,7 +722,7 @@ $while_stmt.tree = $stype; } : WHILE test[expr_contextType.Load] COLON s1=suite (ORELSE COLON s2=suite)? { - $stype = makeWhile($WHILE, (exprType)$test.tree, $s1.stmts, $s2.stmts); + $stype = actions.makeWhile($WHILE, (exprType)$test.tree, $s1.stmts, $s2.stmts); } ; @@ -1080,7 +732,7 @@ $for_stmt.tree = $stype; } : FOR exprlist[expr_contextType.Store] IN testlist[expr_contextType.Load] COLON s1=suite (ORELSE COLON s2=suite)? { - $stype = makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s2.stmts, $s2.stmts); + $stype = actions.makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s2.stmts, $s2.stmts); } ; @@ -1095,10 +747,10 @@ } : TRY COLON trysuite=suite ( e+=except_clause+ (ORELSE COLON elsesuite=suite)? (FINALLY COLON finalsuite=suite)? { - $stype = makeTryExcept($TRY, $trysuite.stmts, $e, $elsesuite.stmts, $finalsuite.stmts); + $stype = actions.makeTryExcept($TRY, $trysuite.stmts, $e, $elsesuite.stmts, $finalsuite.stmts); } | FINALLY COLON finalsuite=suite { - $stype = makeTryFinally($TRY, $trysuite.stmts, $finalsuite.stmts); + $stype = actions.makeTryFinally($TRY, $trysuite.stmts, $finalsuite.stmts); } ) ; @@ -1109,7 +761,7 @@ $with_stmt.tree = $stype; } :WITH test[expr_contextType.Load] (with_var)? COLON suite { - $stype = new With($WITH, (exprType)$test.tree, $with_var.etype, makeStmts($suite.stmts)); + $stype = new With($WITH, (exprType)$test.tree, $with_var.etype, actions.makeStmts($suite.stmts)); } ; @@ -1122,7 +774,7 @@ //except_clause: 'except' [test [',' test]] except_clause : EXCEPT (t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? COLON suite - -> ^(EXCEPT<excepthandlerType>[$EXCEPT, (exprType)$t1.tree, (exprType)$t2.tree, makeStmts($suite.stmts), $EXCEPT.getLine(), $EXCEPT.getCharPositionInLine()]) + -> ^(EXCEPT<excepthandlerType>[$EXCEPT, (exprType)$t1.tree, (exprType)$t2.tree, actions.makeStmts($suite.stmts), $EXCEPT.getLine(), $EXCEPT.getCharPositionInLine()]) ; //suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT @@ -1165,13 +817,57 @@ System.out.println("c.left:" + c.left); c.comparators = new exprType[]{(exprType)c.getChild(1)}; switch (c.getType()) { + case LESS: + c.ops = new cmpopType[]{cmpopType.Lt}; + break; + case GREATER: + c.ops = new cmpopType[]{cmpopType.Gt}; + break; + case EQUAL: + c.ops = new cmpopType[]{cmpopType.Eq}; + break; + case GREATEREQUAL: + c.ops = new cmpopType[]{cmpopType.GtE}; + break; + case LESSEQUAL: + c.ops = new cmpopType[]{cmpopType.LtE}; + break; + case ALT_NOTEQUAL: + c.ops = new cmpopType[]{cmpopType.Eq}; + break; + case NOTEQUAL: + c.ops = new cmpopType[]{cmpopType.NotEq}; + break; case IN: c.ops = new cmpopType[]{cmpopType.In}; break; + case NotIn: + c.ops = new cmpopType[]{cmpopType.NotIn}; + break; + case IS: + c.ops = new cmpopType[]{cmpopType.Is}; + break; + case IsNot: + c.ops = new cmpopType[]{cmpopType.IsNot}; + break; + default: + c.ops = new cmpopType[]{cmpopType.UNDEFINED}; } } } - : expr[ctype] (IN<Compare>^ expr[ctype])* +//comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' + : expr[ctype] (LESS<Compare>^ expr[ctype] + |GREATER<Compare>^ expr[ctype] + |EQUAL<Compare>^ expr[ctype] + |GREATEREQUAL<Compare>^ expr[ctype] + |LESSEQUAL<Compare>^ expr[ctype] + |ALT_NOTEQUAL<Compare>^ expr[ctype] + |NOTEQUAL<Compare>^ expr[ctype] + |IN<Compare>^ expr[ctype] + //|NotIn<Compare>^ exper[ctype] + |IS<Compare>^ expr[ctype] + //|IsNot<Compare>^ exper[ctype] + )* ; //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' @@ -1290,12 +986,12 @@ | LCURLY (dictmaker)? RCURLY -> ^(Dict LCURLY ^(Elts dictmaker)?) | BACKQUOTE testlist[expr_contextType.Load] BACKQUOTE -> ^(Repr BACKQUOTE testlist) | NAME -> ^(NameTok<Name>[$NAME, $NAME.text, $expr::ctype]) - | INT -> ^(NumTok<Num>[$INT, makeInt($INT)]) - | LONGINT -> ^(NumTok<Num>[$LONGINT, makeInt($LONGINT)]) - | FLOAT -> ^(NumTok<Num>[$FLOAT, makeFloat($FLOAT)]) - | COMPLEX -> ^(NumTok<Num>[$COMPLEX, makeComplex($COMPLEX)]) + | INT -> ^(NumTok<Num>[$INT, actions.makeInt($INT)]) + | LONGINT -> ^(NumTok<Num>[$LONGINT, actions.makeInt($LONGINT)]) + | FLOAT -> ^(NumTok<Num>[$FLOAT, actions.makeFloat($FLOAT)]) + | COMPLEX -> ^(NumTok<Num>[$COMPLEX, actions.makeComplex($COMPLEX)]) | (S+=STRING)+ - -> ^(StrTok<Str>[extractStringToken($S), extractStrings($S)]) + -> ^(StrTok<Str>[actions.extractStringToken($S), actions.extractStrings($S)]) ; //listmaker: test ( list_for | (',' test)* [','] ) @@ -1306,7 +1002,7 @@ : t+=test[expr_contextType.Load] ( list_for -> ^(ListComp test list_for) | (options {greedy=true;}:COMMA t+=test[expr_contextType.Load])* { - $etype = new org.python.antlr.ast.List($listmaker.start, makeExprs($t), $expr::ctype); + $etype = new org.python.antlr.ast.List($listmaker.start, actions.makeExprs($t), $expr::ctype); } ) (COMMA)? ; @@ -1315,7 +1011,7 @@ testlist_gexp : t+=test[expr_contextType.Load] ( ((options {k=2;}: c1=COMMA t+=test[expr_contextType.Load])* (c2=COMMA)? - -> { $c1 != null || $c2 != null }? ^(TupleTok<Tuple>[$testlist_gexp.start, makeExprs($t), $expr::ctype]) + -> { $c1 != null || $c2 != null }? ^(TupleTok<Tuple>[$testlist_gexp.start, actions.makeExprs($t), $expr::ctype]) -> test ) | ( gen_for -> ^(GeneratorExp test gen_for) @@ -1329,7 +1025,7 @@ ; //trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME -trailer : LPAREN (arglist -> ^(LPAREN<Call>[$LPAREN, null, makeExprs($arglist.args), makeKeywords($arglist.keywords), $arglist.starargs, $arglist.kwargs]) +trailer : LPAREN (arglist -> ^(LPAREN<Call>[$LPAREN, null, actions.makeExprs($arglist.args), actions.makeKeywords($arglist.keywords), $arglist.starargs, $arglist.kwargs]) | -> ^(LPAREN<Call>[$LPAREN, null, new exprType[0\], new keywordType[0\], null, null]) ) RPAREN @@ -1387,7 +1083,7 @@ //exprlist: expr (',' expr)* [','] exprlist[expr_contextType ctype] returns [exprType etype] : (expr[expr_contextType.Load] COMMA) => e+=expr[ctype] (options {k=2;}: COMMA e+=expr[ctype])* (COMMA)? { - $etype = new Tuple($exprlist.start, makeExprs($e), ctype); + $etype = new Tuple($exprlist.start, actions.makeExprs($e), ctype); } | expr[ctype] { $etype = (exprType)$expr.tree; @@ -1404,7 +1100,7 @@ //testlist: test (',' test)* [','] testlist[expr_contextType ctype] returns [exprType etype] : (test[expr_contextType.Load] COMMA) => t+=test[ctype] (options {k=2;}: c1=COMMA t+=test[ctype])* (c2=COMMA)? { - $etype = new Tuple($testlist.start, makeExprs($t), ctype); + $etype = new Tuple($testlist.start, actions.makeExprs($t), ctype); } | test[ctype] { $etype = (exprType)$test.tree; @@ -1426,7 +1122,7 @@ $classdef.tree = $stype; } :CLASS NAME (LPAREN testlist[expr_contextType.Load]? RPAREN)? COLON suite { - $stype = new ClassDef($CLASS, $NAME.getText(), makeBases($testlist.etype), makeStmts($suite.stmts)); + $stype = new ClassDef($CLASS, $NAME.getText(), actions.makeBases($testlist.etype), actions.makeStmts($suite.stmts)); } ; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-18 18:07:14
|
Revision: 5198 http://jython.svn.sourceforge.net/jython/?rev=5198&view=rev Author: fwierzbicki Date: 2008-08-18 18:07:09 +0000 (Mon, 18 Aug 2008) Log Message: ----------- merge with latest + lots of work on trailer rules. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-18 14:46:22 UTC (rev 5197) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-18 18:07:09 UTC (rev 5198) @@ -56,16 +56,13 @@ * REQUIRES ANTLR v3 * * - * Baby step towards an antlr based Jython parser. - * Terence's Lexer is intact pretty much unchanged, the parser has - * been altered to produce an AST - the AST work started from tne newcompiler - * grammar from Jim Baker minus post-2.3 features. The current parsing - * and compiling strategy looks like this: + * Updated the original parser for Python 2.5 features. The parser has been + * altered to produce an AST - the AST work started from tne newcompiler + * grammar from Jim Baker. The current parsing and compiling strategy looks + * like this: * * Python source->Python.g->simple antlr AST->PythonWalker.g-> * decorated AST (org/python/parser/ast/*)->CodeCompiler(ASM)->.class - * - * for a very limited set of functionality. */ grammar Python; @@ -162,6 +159,8 @@ import org.python.antlr.ast.Assign; import org.python.antlr.ast.Attribute; import org.python.antlr.ast.BinOp; +import org.python.antlr.ast.BoolOp; +import org.python.antlr.ast.boolopType; import org.python.antlr.ast.Break; import org.python.antlr.ast.Call; import org.python.antlr.ast.ClassDef; @@ -195,6 +194,7 @@ import org.python.antlr.ast.Raise; import org.python.antlr.ast.operatorType; import org.python.antlr.ast.Return; +import org.python.antlr.ast.Slice; import org.python.antlr.ast.sliceType; import org.python.antlr.ast.stmtType; import org.python.antlr.ast.Str; @@ -202,8 +202,11 @@ import org.python.antlr.ast.TryExcept; import org.python.antlr.ast.TryFinally; import org.python.antlr.ast.Tuple; +import org.python.antlr.ast.unaryopType; +import org.python.antlr.ast.UnaryOp; import org.python.antlr.ast.While; import org.python.antlr.ast.With; +import org.python.antlr.ast.Yield; import org.python.core.Py; import org.python.core.PyString; import org.python.core.PyUnicode; @@ -272,13 +275,17 @@ * 4] */ +//For use in partial parsing. +public boolean eofWhileNested = false; +public boolean partial = false; + +int implicitLineJoiningLevel = 0; +int startPos=-1; + //If you want to use another error recovery mechanisms change this //and the same one in the parser. private ErrorHandler errorHandler; -int implicitLineJoiningLevel = 0; -int startPos=-1; - public void setErrorHandler(ErrorHandler eh) { this.errorHandler = eh; } @@ -297,6 +304,9 @@ state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { + if (implicitLineJoiningLevel > 0) { + eofWhileNested = true; + } return Token.EOF_TOKEN; } try { @@ -320,9 +330,9 @@ } //single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE -single_input : NEWLINE? -> ^(Interactive) - | simple_stmt -> ^(Interactive simple_stmt) - | compound_stmt NEWLINE -> ^(Interactive compound_stmt) +single_input : NEWLINE* EOF -> ^(Interactive) + | simple_stmt NEWLINE* EOF -> ^(Interactive simple_stmt) + | compound_stmt NEWLINE+ EOF -> ^(Interactive compound_stmt) ; //file_input: (NEWLINE | stmt)* ENDMARKER @@ -332,7 +342,7 @@ ; //eval_input: testlist NEWLINE* ENDMARKER -eval_input : (NEWLINE)* testlist[expr_contextType.Load] (NEWLINE)* -> ^(Expression testlist) +eval_input : LEADING_WS? (NEWLINE)* testlist[expr_contextType.Load] (NEWLINE)* EOF -> ^(Expression testlist) ; //not in CPython's Grammar file @@ -606,7 +616,7 @@ ; //yield_stmt: yield_expr -yield_stmt : yield_expr +yield_stmt : yield_expr -> ^(ExprTok<Expr>[$yield_expr.start, (exprType)$yield_expr.tree]) ; //raise_stmt: 'raise' [test [',' test [',' test]]] @@ -631,14 +641,14 @@ -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeStarAlias($STAR), actions.makeLevel($d)]) | i1=import_as_names -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeAliases($i1.atypes), actions.makeLevel($d)]) - | LPAREN i2=import_as_names RPAREN + | LPAREN i2=import_as_names COMMA? RPAREN -> ^(FROM<ImportFrom>[$FROM, actions.makeFromText($d, $dotted_name.text), actions.makeAliases($i2.atypes), actions.makeLevel($d)]) ) ; //import_as_names: import_as_name (',' import_as_name)* [','] import_as_names returns [aliasType[\] atypes] - : n+=import_as_name (COMMA! n+=import_as_name)* (COMMA!)? { + : n+=import_as_name (COMMA! n+=import_as_name)* { $atypes = (aliasType[])$n.toArray(new aliasType[$n.size()]); } ; @@ -732,7 +742,7 @@ $for_stmt.tree = $stype; } : FOR exprlist[expr_contextType.Store] IN testlist[expr_contextType.Load] COLON s1=suite (ORELSE COLON s2=suite)? { - $stype = actions.makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s2.stmts, $s2.stmts); + $stype = actions.makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s1.stmts, $s2.stmts); } ; @@ -779,42 +789,17 @@ //suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT suite returns [List stmts] - : s+=simple_stmt {$stmts = $s;} + : ss+=simple_stmt {$stmts = $ss;} | NEWLINE! INDENT (s+=stmt)+ DEDENT {$stmts = $s;} ; //test: or_test ['if' or_test 'else' test] | lambdef test[expr_contextType ctype] - :o1=or_test[ctype] - ( (IF or_test[expr_contextType.Load] ORELSE) => IF o2=or_test[ctype] ORELSE test[expr_contextType.Load] - -> ^(IfExp ^(Test $o2) ^(Body $o1) ^(ORELSE test)) - | -> or_test - ) - | lambdef {debug("parsed lambdef");} - ; - -//or_test: and_test ('or' and_test)* -or_test[expr_contextType ctype] : and_test[ctype] (OR^ and_test[ctype])* - ; - -//and_test: not_test ('and' not_test)* -and_test[expr_contextType ctype] : not_test[ctype] (AND^ not_test[ctype])* - ; - -//not_test: 'not' not_test | comparison -not_test[expr_contextType ctype] : NOT^ not_test[ctype] - | comparison[ctype] - ; - -//comparison: expr (comp_op expr)* -comparison[expr_contextType ctype] @after { - if ($comparison.tree instanceof Compare) { - System.out.println("found Compare"); - Compare c = (Compare)$comparison.tree; + if ($test.tree instanceof Compare) { + Compare c = (Compare)$test.tree; //FIXME: Only handling the case of a single comparison. c.left = (exprType)c.getChild(0); - System.out.println("c.left:" + c.left); c.comparators = new exprType[]{(exprType)c.getChild(1)}; switch (c.getType()) { case LESS: @@ -853,8 +838,80 @@ default: c.ops = new cmpopType[]{cmpopType.UNDEFINED}; } + } else if ($test.tree instanceof BoolOp) { + BoolOp b = (BoolOp)$test.tree; + List values = new ArrayList(); + + exprType left = (exprType)b.getChild(0); + exprType right = (exprType)b.getChild(1); + + exprType[] e; + if (left.getType() == b.getType() && right.getType() == b.getType()) { + BoolOp leftB = (BoolOp)left; + BoolOp rightB = (BoolOp)right; + int lenL = leftB.values.length; + int lenR = rightB.values.length; + e = new exprType[lenL + lenR]; + System.arraycopy(leftB.values, 0, e, 0, lenL - 1); + System.arraycopy(rightB.values, 0, e, lenL - 1, lenL + lenR); + } else if (left.getType() == b.getType()) { + BoolOp leftB = (BoolOp)left; + e = new exprType[leftB.values.length + 1]; + System.arraycopy(leftB.values, 0, e, 0, leftB.values.length); + e[e.length - 1] = right; + } else if (right.getType() == b.getType()) { + BoolOp rightB = (BoolOp)right; + e = new exprType[rightB.values.length + 1]; + System.arraycopy(rightB.values, 0, e, 0, rightB.values.length); + e[e.length - 1] = left; + } else { + e = new exprType[2]; + e[0] = left; + e[1] = right; + } + b.values = e; + switch (b.getType()) { + case AND: + b.op = boolopType.And; + break; + case OR: + b.op = boolopType.Or; + break; + default: + b.op = boolopType.UNDEFINED; + } } } + + :o1=or_test[ctype] + ( (IF or_test[expr_contextType.Load] ORELSE) => IF o2=or_test[ctype] ORELSE test[expr_contextType.Load] + -> ^(IfExp ^(Test $o2) ^(Body $o1) ^(ORELSE test)) + | -> or_test + ) + | lambdef {debug("parsed lambdef");} + ; + +//or_test: and_test ('or' and_test)* +or_test[expr_contextType ctype] : and_test[ctype] (OR<BoolOp>^ and_test[ctype])* + ; + +//and_test: not_test ('and' not_test)* +and_test[expr_contextType ctype] : not_test[ctype] (AND<BoolOp>^ not_test[ctype])* + ; + +//not_test: 'not' not_test | comparison +not_test[expr_contextType ctype] returns [exprType etype] +@after { + if ($etype != null) { + $not_test.tree = $etype; + } +} + : NOT nt=not_test[ctype] {$etype = new UnaryOp($NOT, unaryopType.Not, (exprType)$nt.tree);} + | comparison[ctype] + ; + +//comparison: expr (comp_op expr)* +comparison[expr_contextType ctype] //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' : expr[ctype] (LESS<Compare>^ expr[ctype] |GREATER<Compare>^ expr[ctype] @@ -904,22 +961,52 @@ case MINUS: b.op = operatorType.Sub; break; + case STAR: + b.op = operatorType.Mult; + break; + case SLASH: + b.op = operatorType.Div; + break; + case PERCENT: + b.op = operatorType.Mod; + break; + case DOUBLESLASH: + b.op = operatorType.FloorDiv; + break; + case AMPER: + b.op = operatorType.BitAnd; + break; + case VBAR: + b.op = operatorType.BitOr; + break; + case CIRCUMFLEX: + b.op = operatorType.BitXor; + break; + case LEFTSHIFT: + b.op = operatorType.LShift; + break; + case RIGHTSHIFT: + b.op = operatorType.RShift; + break; + case DOUBLESTAR: + b.op = operatorType.Pow; + break; } } } - : xor_expr (VBAR^ xor_expr)* + : xor_expr (VBAR<BinOp>^ xor_expr)* ; //xor_expr: and_expr ('^' and_expr)* -xor_expr : and_expr (CIRCUMFLEX^ and_expr)* +xor_expr : and_expr (CIRCUMFLEX<BinOp>^ and_expr)* ; //and_expr: shift_expr ('&' shift_expr)* -and_expr : shift_expr (AMPER^ shift_expr)* +and_expr : shift_expr (AMPER<BinOp>^ shift_expr)* ; //shift_expr: arith_expr (('<<'|'>>') arith_expr)* -shift_expr : arith_expr ((LEFTSHIFT^|RIGHTSHIFT^) arith_expr)* +shift_expr : arith_expr ((LEFTSHIFT<BinOp>^|RIGHTSHIFT<BinOp>^) arith_expr)* ; //arith_expr: term (('+'|'-') term)* @@ -929,15 +1016,19 @@ ; //term: factor (('*'|'/'|'%'|'//') factor)* -term : factor ((STAR^ | SLASH^ | PERCENT^ | DOUBLESLASH^ ) factor)* +term : factor ((STAR<BinOp>^ | SLASH<BinOp>^ | PERCENT<BinOp>^ | DOUBLESLASH<BinOp>^ ) factor)* ; //factor: ('+'|'-'|'~') factor | power -factor : PLUS factor -> ^(UAdd PLUS factor) - | MINUS factor -> ^(USub MINUS factor) - | TILDE factor -> ^(Invert TILDE factor) - | power - ; +factor returns [exprType etype] +@after { + $factor.tree = $etype; +} + : PLUS p=factor {$etype = new UnaryOp($PLUS, unaryopType.UAdd, $p.etype);} + | MINUS m=factor {$etype = actions.negate($MINUS, $m.etype);} + | TILDE t=factor {$etype = new UnaryOp($TILDE, unaryopType.Invert, $t.etype);} + | power {$etype = (exprType)$power.tree;} + ; //power: atom trailer* ['**' factor] power returns [exprType etype] @@ -949,8 +1040,12 @@ : atom (t+=trailer)* (options {greedy=true;}:DOUBLESTAR factor)? { if ($t != null) { exprType current = (exprType)$atom.tree; - for(int i = $t.size() - 1; i > -1; i--) { + //for(int i = $t.size() - 1; i > -1; i--) { + for(int i = 0; i < $t.size(); i++) { Object o = $t.get(i); + if (current instanceof Context) { + ((Context)current).setContext(expr_contextType.Load); + } //XXX: good place for an interface to avoid all of this instanceof if (o instanceof Call) { Call c = (Call)o; @@ -960,6 +1055,10 @@ Subscript c = (Subscript)o; c.value = current; current = c; + } else if (o instanceof Attribute) { + Attribute c = (Attribute)o; + c.value = current; + current = c; } } $etype = (exprType)current; @@ -973,7 +1072,7 @@ // '`' testlist1 '`' | // NAME | NUMBER | STRING+) atom : LPAREN - ( yield_expr -> ^(Parens LPAREN yield_expr) + ( yield_expr -> yield_expr | testlist_gexp -> testlist_gexp | -> ^(TupleTok<Tuple>[$LPAREN, new exprType[0\], $expr::ctype]) ) @@ -1030,10 +1129,11 @@ ) RPAREN | LBRACK s=subscriptlist RBRACK -> $s - | DOT^ attr {debug("motched DOT^ NAME");} + | DOT attr -> ^(DOT<Attribute>[$DOT, null, $attr.text, expr_contextType.Load]) ; //subscriptlist: subscript (',' subscript)* [','] +//FIXME: tuples not always created when commas are present. subscriptlist returns [exprType etype] @after { $subscriptlist.tree = $etype; @@ -1070,11 +1170,21 @@ ; //subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] -subscript : DOT DOT DOT -> Ellipsis - | (test[expr_contextType.Load] COLON) => t1=test[expr_contextType.Load] (COLON (t2=test[expr_contextType.Load])? (sliceop)?)? -> ^(Subscript ^(Lower $t1) ^(Upper COLON ^(UpperOp $t2)?)? sliceop?) - | (COLON) => COLON (test[expr_contextType.Load])? (sliceop)? -> ^(Subscript ^(Upper COLON ^(UpperOp test)?)? sliceop?) - | test[expr_contextType.Load] -> ^(Subscript<Index>[$test.start, (exprType)$test.tree]) - ; +subscript returns [sliceType sltype] +@after { + if ($sltype != null) { + $subscript.tree = $sltype; + } +} + : DOT DOT DOT -> Ellipsis + | (test[expr_contextType.Load] COLON) => lower=test[expr_contextType.Load] (c1=COLON (upper=test[expr_contextType.Load])? (sliceop)?)? { + $sltype = actions.makeSubscript($lower.tree, $c1, $upper.tree, $sliceop.tree); + } + | (COLON) => c2=COLON (test[expr_contextType.Load])? (sliceop)? { + $sltype = actions.makeSubscript(null, $c2, $upper.tree, $sliceop.tree); + } + | test[expr_contextType.Load] -> ^(Subscript<Index>[$test.start, (exprType)$test.tree]) + ; //sliceop: ':' [test] sliceop : COLON (test[expr_contextType.Load])? -> ^(Step COLON ^(StepOp test)?) @@ -1195,7 +1305,7 @@ //yield_expr: 'yield' [testlist] yield_expr : YIELD testlist[expr_contextType.Load]? - -> ^(YIELD ^(Value testlist)?) + -> ^(YIELD<Yield>[$YIELD, $testlist.etype]) ; //XXX: @@ -1377,6 +1487,13 @@ } ; +STRINGPART + : {partial}?=> ('r'|'u'|'ur'|'R'|'U'|'UR'|'uR'|'Ur')? + ( '\'\'\'' ~('\'\'\'')* + | '"""' ~('"""')* + ) + ; + /** the two '"'? cause a warning -- is there a way to avoid that? */ fragment TRIQUOTE This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-19 03:11:15
|
Revision: 5199 http://jython.svn.sourceforge.net/jython/?rev=5199&view=rev Author: fwierzbicki Date: 2008-08-19 03:11:13 +0000 (Tue, 19 Aug 2008) Log Message: ----------- Compare works better -- grammar is approaching usefullness. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-18 18:07:09 UTC (rev 5198) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:11:13 UTC (rev 5199) @@ -103,8 +103,6 @@ ExceptHandler; StrTok; NumTok; - IsNot; - NotIn; Type; Inst; Tback; @@ -144,6 +142,7 @@ ListIf; Parens; Brackets; + } @header { @@ -796,49 +795,7 @@ //test: or_test ['if' or_test 'else' test] | lambdef test[expr_contextType ctype] @after { - if ($test.tree instanceof Compare) { - Compare c = (Compare)$test.tree; - //FIXME: Only handling the case of a single comparison. - c.left = (exprType)c.getChild(0); - c.comparators = new exprType[]{(exprType)c.getChild(1)}; - switch (c.getType()) { - case LESS: - c.ops = new cmpopType[]{cmpopType.Lt}; - break; - case GREATER: - c.ops = new cmpopType[]{cmpopType.Gt}; - break; - case EQUAL: - c.ops = new cmpopType[]{cmpopType.Eq}; - break; - case GREATEREQUAL: - c.ops = new cmpopType[]{cmpopType.GtE}; - break; - case LESSEQUAL: - c.ops = new cmpopType[]{cmpopType.LtE}; - break; - case ALT_NOTEQUAL: - c.ops = new cmpopType[]{cmpopType.Eq}; - break; - case NOTEQUAL: - c.ops = new cmpopType[]{cmpopType.NotEq}; - break; - case IN: - c.ops = new cmpopType[]{cmpopType.In}; - break; - case NotIn: - c.ops = new cmpopType[]{cmpopType.NotIn}; - break; - case IS: - c.ops = new cmpopType[]{cmpopType.Is}; - break; - case IsNot: - c.ops = new cmpopType[]{cmpopType.IsNot}; - break; - default: - c.ops = new cmpopType[]{cmpopType.UNDEFINED}; - } - } else if ($test.tree instanceof BoolOp) { + if ($test.tree instanceof BoolOp) { BoolOp b = (BoolOp)$test.tree; List values = new ArrayList(); @@ -911,20 +868,31 @@ ; //comparison: expr (comp_op expr)* -comparison[expr_contextType ctype] //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' - : expr[ctype] (LESS<Compare>^ expr[ctype] - |GREATER<Compare>^ expr[ctype] - |EQUAL<Compare>^ expr[ctype] - |GREATEREQUAL<Compare>^ expr[ctype] - |LESSEQUAL<Compare>^ expr[ctype] - |ALT_NOTEQUAL<Compare>^ expr[ctype] - |NOTEQUAL<Compare>^ expr[ctype] - |IN<Compare>^ expr[ctype] - //|NotIn<Compare>^ exper[ctype] - |IS<Compare>^ expr[ctype] - //|IsNot<Compare>^ exper[ctype] - )* +comparison[expr_contextType ctype] +@init { + List cmps = new ArrayList(); +} +@after { + if (!cmps.isEmpty()) { + $comparison.tree = new Compare($left.tree, (exprType)$left.tree, actions.makeCmpOps(cmps), actions.makeExprs($right)); + } +} + : left=expr[ctype] + ( ( LESS right+=expr[ctype] {cmps.add(cmpopType.Lt);} + |GREATER right+=expr[ctype] {cmps.add(cmpopType.Gt);} + |EQUAL right+=expr[ctype] {cmps.add(cmpopType.Eq);} + |GREATEREQUAL right+=expr[ctype] {cmps.add(cmpopType.GtE);} + |LESSEQUAL right+=expr[ctype] {cmps.add(cmpopType.LtE);} + |ALT_NOTEQUAL right+=expr[ctype] {cmps.add(cmpopType.NotEq);} + |NOTEQUAL right+=expr[ctype] {cmps.add(cmpopType.NotEq);} + |IN right+=expr[ctype] {cmps.add(cmpopType.In);} + |IS right+=expr[ctype] {cmps.add(cmpopType.Is);} + |NOT IN right+=expr[ctype] {cmps.add(cmpopType.NotIn);} + |IS NOT right+=expr[ctype] {cmps.add(cmpopType.IsNot);} + )+ + | -> $left + ) ; //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' @@ -936,11 +904,12 @@ | ALT_NOTEQUAL | NOTEQUAL | IN - | NOT IN -> NotIn +// | NOT IN -> NotIn | IS - | IS NOT -> IsNot +// | IS NOT -> IsNot ; + //expr: xor_expr ('|' xor_expr)* expr[expr_contextType ect] scope { @@ -1079,7 +1048,7 @@ RPAREN | LBRACK (listmaker -> listmaker - | -> ^(Brackets LBRACK ^(ListTok)) + | -> ^(LBRACK<org.python.antlr.ast.List>[$LBRACK, new exprType[0\], $expr::ctype]) ) RBRACK | LCURLY (dictmaker)? RCURLY -> ^(Dict LCURLY ^(Elts dictmaker)?) @@ -1129,7 +1098,7 @@ ) RPAREN | LBRACK s=subscriptlist RBRACK -> $s - | DOT attr -> ^(DOT<Attribute>[$DOT, null, $attr.text, expr_contextType.Load]) + | DOT attr -> ^(DOT<Attribute>[$DOT, null, $attr.text, $expr::ctype]) ; //subscriptlist: subscript (',' subscript)* [','] @@ -1177,11 +1146,11 @@ } } : DOT DOT DOT -> Ellipsis - | (test[expr_contextType.Load] COLON) => lower=test[expr_contextType.Load] (c1=COLON (upper=test[expr_contextType.Load])? (sliceop)?)? { - $sltype = actions.makeSubscript($lower.tree, $c1, $upper.tree, $sliceop.tree); + | (test[expr_contextType.Load] COLON) => lower=test[expr_contextType.Load] (c1=COLON (upper1=test[expr_contextType.Load])? (sliceop)?)? { + $sltype = actions.makeSubscript($lower.tree, $c1, $upper1.tree, $sliceop.tree); } - | (COLON) => c2=COLON (test[expr_contextType.Load])? (sliceop)? { - $sltype = actions.makeSubscript(null, $c2, $upper.tree, $sliceop.tree); + | (COLON) => c2=COLON (upper2=test[expr_contextType.Load])? (sliceop)? { + $sltype = actions.makeSubscript(null, $c2, $upper2.tree, $sliceop.tree); } | test[expr_contextType.Load] -> ^(Subscript<Index>[$test.start, (exprType)$test.tree]) ; @@ -1209,6 +1178,9 @@ //testlist: test (',' test)* [','] testlist[expr_contextType ctype] returns [exprType etype] +@after { + $testlist.tree = $etype; +} : (test[expr_contextType.Load] COMMA) => t+=test[ctype] (options {k=2;}: c1=COMMA t+=test[ctype])* (c2=COMMA)? { $etype = new Tuple($testlist.start, actions.makeExprs($t), ctype); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-19 03:21:54
|
Revision: 5200 http://jython.svn.sourceforge.net/jython/?rev=5200&view=rev Author: fwierzbicki Date: 2008-08-19 03:21:51 +0000 (Tue, 19 Aug 2008) Log Message: ----------- factor comp_op back out. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:11:13 UTC (rev 5199) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:21:51 UTC (rev 5200) @@ -879,35 +879,26 @@ } } : left=expr[ctype] - ( ( LESS right+=expr[ctype] {cmps.add(cmpopType.Lt);} - |GREATER right+=expr[ctype] {cmps.add(cmpopType.Gt);} - |EQUAL right+=expr[ctype] {cmps.add(cmpopType.Eq);} - |GREATEREQUAL right+=expr[ctype] {cmps.add(cmpopType.GtE);} - |LESSEQUAL right+=expr[ctype] {cmps.add(cmpopType.LtE);} - |ALT_NOTEQUAL right+=expr[ctype] {cmps.add(cmpopType.NotEq);} - |NOTEQUAL right+=expr[ctype] {cmps.add(cmpopType.NotEq);} - |IN right+=expr[ctype] {cmps.add(cmpopType.In);} - |IS right+=expr[ctype] {cmps.add(cmpopType.Is);} - |NOT IN right+=expr[ctype] {cmps.add(cmpopType.NotIn);} - |IS NOT right+=expr[ctype] {cmps.add(cmpopType.IsNot);} + ( ( comp_op right+=expr[ctype] {cmps.add($comp_op.op);} )+ | -> $left ) ; //comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' -comp_op : LESS - | GREATER - | EQUAL - | GREATEREQUAL - | LESSEQUAL - | ALT_NOTEQUAL - | NOTEQUAL - | IN -// | NOT IN -> NotIn - | IS -// | IS NOT -> IsNot - ; +comp_op returns [cmpopType op] + : LESS {$op = cmpopType.Lt;} + | GREATER {$op = cmpopType.Gt;} + | EQUAL {$op = cmpopType.Eq;} + | GREATEREQUAL {$op = cmpopType.GtE;} + | LESSEQUAL {$op = cmpopType.LtE;} + | ALT_NOTEQUAL {$op = cmpopType.NotEq;} + | NOTEQUAL {$op = cmpopType.NotEq;} + | IN {$op = cmpopType.In;} + | NOT IN {$op = cmpopType.NotIn;} + | IS {$op = cmpopType.Is;} + | IS NOT {$op = cmpopType.IsNot;} + ; //expr: xor_expr ('|' xor_expr)* This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-19 03:41:03
|
Revision: 5201 http://jython.svn.sourceforge.net/jython/?rev=5201&view=rev Author: fwierzbicki Date: 2008-08-19 03:41:01 +0000 (Tue, 19 Aug 2008) Log Message: ----------- make augassign work. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:21:51 UTC (rev 5200) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:41:01 UTC (rev 5201) @@ -157,6 +157,7 @@ import org.python.antlr.ast.Assert; import org.python.antlr.ast.Assign; import org.python.antlr.ast.Attribute; +import org.python.antlr.ast.AugAssign; import org.python.antlr.ast.BinOp; import org.python.antlr.ast.BoolOp; import org.python.antlr.ast.boolopType; @@ -504,32 +505,46 @@ //expr_stmt: testlist (augassign (yield_expr|testlist) | // ('=' (yield_expr|testlist))*) -expr_stmt : ((testlist[expr_contextType.Load] (ASSIGN|augassign)) => lhs=testlist[expr_contextType.Store] - | lhs=testlist[expr_contextType.Load] - ) - ( (augassign yield_expr -> ^(augassign $lhs yield_expr)) - | (augassign rhs=testlist[expr_contextType.Load] -> ^(augassign $lhs $rhs)) - | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(AssignTok<Assign>[$at, actions.makeAssignTargets((exprType)$lhs.tree, $t), actions.makeAssignValue($t)])) - | ((ay=ASSIGN y+=yield_expr)+ -> ^(AssignTok<Assign>[$ay, actions.makeAssignTargets((exprType)$lhs.tree, $y), actions.makeAssignValue($y)])) - | -> ExprTok<Expr>[$lhs.start, (exprType)$lhs.tree] - ) - ; +expr_stmt +@init { + stmtType stype = null; +} +@after { + if (stype != null) { + $expr_stmt.tree = stype; + } +} + : + ((testlist[expr_contextType.Load] augassign) => lhs=testlist[expr_contextType.AugStore] + ( (aay=augassign y1=yield_expr {stype = new AugAssign($lhs.tree, (exprType)$lhs.tree, $aay.op, (exprType)$y1.tree);}) + | (aat=augassign rhs=testlist[expr_contextType.Load] {stype = new AugAssign($lhs.tree, (exprType)$lhs.tree, $aat.op, (exprType)$rhs.tree);}) + ) + |(testlist[expr_contextType.Load] ASSIGN) => lhs=testlist[expr_contextType.Store] + ( + | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(AssignTok<Assign>[$at, actions.makeAssignTargets((exprType)$lhs.tree, $t), actions.makeAssignValue($t)])) + | ((ay=ASSIGN y2+=yield_expr)+ -> ^(AssignTok<Assign>[$ay, actions.makeAssignTargets((exprType)$lhs.tree, $y2), actions.makeAssignValue($y2)])) + ) + | lhs=testlist[expr_contextType.Load] -> ExprTok<Expr>[$lhs.start, (exprType)$lhs.tree] + ) + ; + //augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | // '<<=' | '>>=' | '**=' | '//=') -augassign : PLUSEQUAL - | MINUSEQUAL - | STAREQUAL - | SLASHEQUAL - | PERCENTEQUAL - | AMPEREQUAL - | VBAREQUAL - | CIRCUMFLEXEQUAL - | LEFTSHIFTEQUAL - | RIGHTSHIFTEQUAL - | DOUBLESTAREQUAL - | DOUBLESLASHEQUAL - ; +augassign returns [operatorType op] + : PLUSEQUAL {$op = operatorType.Add;} + | MINUSEQUAL {$op = operatorType.Sub;} + | STAREQUAL {$op = operatorType.Mult;} + | SLASHEQUAL {$op = operatorType.Div;} + | PERCENTEQUAL {$op = operatorType.Mod;} + | AMPEREQUAL {$op = operatorType.BitAnd;} + | VBAREQUAL {$op = operatorType.BitOr;} + | CIRCUMFLEXEQUAL {$op = operatorType.BitXor;} + | LEFTSHIFTEQUAL {$op = operatorType.LShift;} + | RIGHTSHIFTEQUAL {$op = operatorType.RShift;} + | DOUBLESTAREQUAL {$op = operatorType.Pow;} + | DOUBLESLASHEQUAL {$op = operatorType.FloorDiv;} + ; //print_stmt: 'print' ( [ test (',' test)* [','] ] | // '>>' test [ (',' test)+ [','] ] ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <fwi...@us...> - 2008-08-19 14:04:16
|
Revision: 5202 http://jython.svn.sourceforge.net/jython/?rev=5202&view=rev Author: fwierzbicki Date: 2008-08-19 14:04:14 +0000 (Tue, 19 Aug 2008) Log Message: ----------- Big cleanup -- reduced imaginary nodes and return values. Modified Paths: -------------- trunk/sandbox/wierzbicki/backup/Python.g Modified: trunk/sandbox/wierzbicki/backup/Python.g =================================================================== --- trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 03:41:01 UTC (rev 5201) +++ trunk/sandbox/wierzbicki/backup/Python.g 2008-08-19 14:04:14 UTC (rev 5202) @@ -75,73 +75,31 @@ INDENT; DEDENT; - ModuleTok; + PYNODE; Interactive; Expression; - ExprTok; - NameTok; Test; - Msg; - Level; Body; - Bases; - FunctionDefTok; - ArgumentsTok; - Args; Arg; - Keyword; - StarArgs; - KWArgs; - AssignTok; AugAssign; - TupleTok; - ListTok; Dict; IfExp; - TryExcept; - TryFinally; - ExceptHandler; - StrTok; - NumTok; - Type; - Inst; - Tback; - Globals; - Locals; Ellipsis; ListComp; Repr; - Subscript; Index; Target; - Value; - Lower; - Upper; Step; - UAdd; - USub; - Invert; - Alias; - Asname; - Decorators; GeneratorExp; Ifs; Elts; - CallTok; - Dest; - Values; - Newline; - FpList; StepOp; - UpperOp; GenFor; GenIf; ListFor; ListIf; - Parens; - Brackets; } @@ -337,8 +295,8 @@ //file_input: (NEWLINE | stmt)* ENDMARKER file_input - : (NEWLINE | s+=stmt)+ -> ^(ModuleTok<Module>[$file_input.start, actions.makeStmts($s)]) - | -> ^(ModuleTok<Module>[$file_input.start, new stmtType[0\]]) + : (NEWLINE | s+=stmt)+ -> ^(PYNODE<Module>[$file_input.start, actions.makeStmts($s)]) + | -> ^(PYNODE<Module>[$file_input.start, new stmtType[0\]]) ; //eval_input: testlist NEWLINE* ENDMARKER @@ -468,7 +426,7 @@ //fpdef: NAME | '(' fplist ')' fpdef[expr_contextType ctype] : NAME - -> ^(NameTok<Name>[$NAME, $NAME.text, ctype]) + -> ^(PYNODE<Name>[$NAME, $NAME.text, ctype]) | (LPAREN fpdef[expr_contextType.Load] COMMA) => LPAREN fplist RPAREN -> fplist | LPAREN fplist RPAREN @@ -522,10 +480,10 @@ ) |(testlist[expr_contextType.Load] ASSIGN) => lhs=testlist[expr_contextType.Store] ( - | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(AssignTok<Assign>[$at, actions.makeAssignTargets((exprType)$lhs.tree, $t), actions.makeAssignValue($t)])) - | ((ay=ASSIGN y2+=yield_expr)+ -> ^(AssignTok<Assign>[$ay, actions.makeAssignTargets((exprType)$lhs.tree, $y2), actions.makeAssignValue($y2)])) + | ((at=ASSIGN t+=testlist[expr_contextType.Store])+ -> ^(PYNODE<Assign>[$at, actions.makeAssignTargets((exprType)$lhs.tree, $t), actions.makeAssignValue($t)])) + | ((ay=ASSIGN y2+=yield_expr)+ -> ^(PYNODE<Assign>[$ay, actions.makeAssignTargets((exprType)$lhs.tree, $y2), actions.makeAssignValue($y2)])) ) - | lhs=testlist[expr_contextType.Load] -> ExprTok<Expr>[$lhs.start, (exprType)$lhs.tree] + | lhs=testlist[expr_contextType.Load] -> PYNODE<Expr>[$lhs.start, (exprType)$lhs.tree] ) ; @@ -630,7 +588,7 @@ ; //yield_stmt: yield_expr -yield_stmt : yield_expr -> ^(ExprTok<Expr>[$yield_expr.start, (exprType)$yield_expr.tree]) +yield_stmt : yield_expr -> ^(PYNODE<Expr>[$yield_expr.start, (exprType)$yield_expr.tree]) ; //raise_stmt: 'raise' [test [',' test [',' test]]] @@ -706,12 +664,15 @@ ; //exec_stmt: 'exec' expr ['in' test [',' test]] -exec_stmt returns [stmtType stype] +exec_stmt +@init { + stmtType stype = null; +} @after { - $exec_stmt.tree = $stype; + $exec_stmt.tree = stype; } : EXEC expr[expr_contextType.Load] (IN t1=test[expr_contextType.Load] (COMMA t2=test[expr_contextType.Load])?)? { - $stype = new Exec($expr.start, (exprType)$expr.tree, (exprType)$t1.tree, (exprType)$t2.tree); + stype = new Exec($expr.start, (exprType)$expr.tree, (exprType)$t1.tree, (exprType)$t2.tree); } ; @@ -741,22 +702,28 @@ ; //while_stmt: 'while' test ':' suite ['else' ':' suite] -while_stmt returns [stmtType stype] +while_stmt +@init { + stmtType stype = null; +} @after { - $while_stmt.tree = $stype; + $while_stmt.tree = stype; } : WHILE test[expr_contextType.Load] COLON s1=suite (ORELSE COLON s2=suite)? { - $stype = actions.makeWhile($WHILE, (exprType)$test.tree, $s1.stmts, $s2.stmts); + stype = actions.makeWhile($WHILE, (exprType)$test.tree, $s1.stmts, $s2.stmts); } ; //for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] -for_stmt returns [stmtType stype] +for_stmt +@init { + stmtType stype = null; +} @after { - $for_stmt.tree = $stype; + $for_stmt.tree = stype; } : FOR exprlist[expr_contextType.Store] IN testlist[expr_contextType.Load] COLON s1=suite (ORELSE COLON s2=suite)? { - $stype = actions.makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s1.stmts, $s2.stmts); + stype = actions.makeFor($FOR, $exprlist.etype, (exprType)$testlist.tree, $s1.stmts, $s2.stmts); } ; @@ -765,27 +732,33 @@ // ['else' ':' suite] // ['finally' ':' suite] | // 'finally' ':' suite)) -try_stmt returns [stmtType stype] +try_stmt +@init { + stmtType stype = null; +} @after { - $try_stmt.tree = $stype; + $try_stmt.tree = stype; } : TRY COLON trysuite=suite ( e+=except_clause+ (ORELSE COLON elsesuite=suite)? (FINALLY COLON finalsuite=suite)? { - $stype = actions.makeTryExcept($TRY, $trysuite.stmts, $e, $elsesuite.stmts, $finalsuite.stmts); + stype = actions.makeTryExcept($TRY, $trysuite.stmts, $e, $elsesuite.stmts, $finalsuite.stmts); } | FINALLY COLON finalsuite=suite { - $stype = actions.makeTryFinally($TRY, $trysuite.stmts, $finalsuite.stmts); + stype = actions.makeTryFinally($TRY, $trysuite.stmts, $finalsuite.stmts); } ) ; //with_stmt: 'with' test [ with_var ] ':' suite -with_stmt returns [stmtType stype] +with_stmt +@init { + stmtType stype = null; +} @after { - $with_stmt.tree = $stype; + $with_stmt.tree = stype; } :WITH test[expr_contextType.Load] (with_var)? COLON suite { - $stype = new With($WITH, (exprType)$test.tree, $with_var.etype, actions.makeStmts($suite.stmts)); + stype = new With($WITH, (exprType)$test.tree, $with_var.etype, actions.makeStmts($suite.stmts)); } ; @@ -1049,7 +1022,7 @@ atom : LPAREN ( yield_expr -> yield_expr | testlist_gexp -> testlist_gexp - | -> ^(TupleTok<Tuple>[$LPAREN, new exprType[0\], $expr::ctype]) + | -> ^(PYNODE<Tuple>[$LPAREN, new exprType[0\], $expr::ctype]) ) RPAREN | LBRACK @@ -1059,13 +1032,13 @@ RBRACK | LCURLY (dictmaker)? RCURLY -> ^(Dict LCURLY ^(Elts dictmaker)?) | BACKQUOTE testlist[expr_contextType.Load] BACKQUOTE -> ^(Repr BACKQUOTE testlist) - | NAME -> ^(NameTok<Name>[$NAME, $NAME.text, $expr::ctype]) - | INT -> ^(NumTok<Num>[$INT, actions.makeInt($INT)]) - | LONGINT -> ^(NumTok<Num>[$LONGINT, actions.makeInt($LONGINT)]) - | FLOAT -> ^(NumTok<Num>[$FLOAT, actions.makeFloat($FLOAT)]) - | COMPLEX -> ^(NumTok<Num>[$COMPLEX, actions.makeComplex($COMPLEX)]) + | NAME -> ^(PYNODE<Name>[$NAME, $NAME.text, $expr::ctype]) + | INT -> ^(PYNODE<Num>[$INT, actions.makeInt($INT)]) + | LONGINT -> ^(PYNODE<Num>[$LONGINT, actions.makeInt($LONGINT)]) + | FLOAT -> ^(PYNODE<Num>[$FLOAT, actions.makeFloat($FLOAT)]) + | COMPLEX -> ^(PYNODE<Num>[$COMPLEX, actions.makeComplex($COMPLEX)]) | (S+=STRING)+ - -> ^(StrTok<Str>[actions.extractStringToken($S), actions.extractStrings($S)]) + -> ^(PYNODE<Str>[actions.extractStringToken($S), actions.extractStrings($S)]) ; //listmaker: test ( list_for | (',' test)* [','] ) @@ -1085,7 +1058,7 @@ testlist_gexp : t+=test[expr_contextType.Load] ( ((options {k=2;}: c1=COMMA t+=test[expr_contextType.Load])* (c2=COMMA)? - -> { $c1 != null || $c2 != null }? ^(TupleTok<Tuple>[$testlist_gexp.start, actions.makeExprs($t), $expr::ctype]) + -> { $c1 != null || $c2 != null }? ^(PYNODE<Tuple>[$testlist_gexp.start, actions.makeExprs($t), $expr::ctype]) -> test ) | ( gen_for -> ^(GeneratorExp test gen_for) @@ -1158,7 +1131,7 @@ | (COLON) => c2=COLON (upper2=test[expr_contextType.Load])? (sliceop)? { $sltype = actions.makeSubscript(null, $c2, $upper2.tree, $sliceop.tree); } - | test[expr_contextType.Load] -> ^(Subscript<Index>[$test.start, (exprType)$test.tree]) + | test[expr_contextType.Load] -> ^(PYNODE<Index>[$test.start, (exprType)$test.tree]) ; //sliceop: ':' [test] @@ -1205,12 +1178,15 @@ ; //classdef: 'class' NAME ['(' [testlist] ')'] ':' suite -classdef returns [stmtType stype] +classdef +@init { + stmtType stype = null; +} @after { - $classdef.tree = $stype; + $classdef.tree = stype; } :CLASS NAME (LPAREN testlist[expr_contextType.Load]? RPAREN)? COLON suite { - $stype = new ClassDef($CLASS, $NAME.getText(), actions.makeBases($testlist.etype), actions.makeStmts($suite.stmts)); + stype = new ClassDef($CLASS, $NAME.getText(), actions.makeBases($testlist.etype), actions.makeStmts($suite.stmts)); } ; This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |