From: <fwi...@us...> - 2008-04-11 00:09:29
|
Revision: 4332 http://jython.svn.sourceforge.net/jython/?rev=4332&view=rev Author: fwierzbicki Date: 2008-04-10 17:09:27 -0700 (Thu, 10 Apr 2008) Log Message: ----------- The start of an _ast module. It relies on the new antlr parser which is now in trunk. For now you must put antlr-runtime-3.0.1 (which is in the extlibs dir) in your classpath to use it. Using it looks like this: >>> import _ast >>> compile("a=1", "<unknown>", "exec", _ast.PyCF_ONLY_AST) tree: (Module (Assign (Target (Name a)) (Value (Num 1)))) Which outputs a tree representation for now. If you do not use _ast or the PyCF_ONLY_AST flag, you should not need antlr-runtime-3.0.1.jar in your path. Modified Paths: -------------- trunk/jython/src/org/python/antlr/PythonTreeWalker.java trunk/jython/src/org/python/core/CompilerFlags.java trunk/jython/src/org/python/core/Py.java trunk/jython/src/org/python/core/PyTableCode.java trunk/jython/src/org/python/core/__builtin__.java trunk/jython/src/org/python/util/PythonInterpreter.java trunk/jython/src/org/python/util/jython.java Added Paths: ----------- trunk/jython/Lib/_ast.py trunk/jython/src/org/python/antlr/IParserHost.java trunk/jython/src/org/python/antlr/PythonGrammar.java trunk/jython/src/org/python/core/antlr.java Added: trunk/jython/Lib/_ast.py =================================================================== --- trunk/jython/Lib/_ast.py (rev 0) +++ trunk/jython/Lib/_ast.py 2008-04-11 00:09:27 UTC (rev 4332) @@ -0,0 +1,21 @@ +from org.python.antlr.ast.boolopType import And,Or +from org.python.antlr.ast.operatorType import Add,Sub,Mult,Div,FloorDiv,Mod,LShift,RShift,BitOr,BitAnd,BitXor +from org.python.antlr.ast.cmpopType import Eq,Gt,GtE,In,Is,IsNot,Lt,LtE,NotEq,NotIn +from org.python.antlr.ast.unaryopType import Invert,Not,UAdd,USub +from org.python.core.PyTableCode import PyCF_ONLY_AST + +import org.python.antlr.ast.aliasType as alias +import org.python.antlr.ast.argumentsType as arguments +import org.python.antlr.ast.boolopType as boolop +import org.python.antlr.ast.cmpopType as cmpop +import org.python.antlr.ast.comprehensionType as comprehension +import org.python.antlr.ast.excepthandlerType as excepthandler +import org.python.antlr.ast.exprType as expr +import org.python.antlr.ast.expr_contextType as expr_context +import org.python.antlr.ast.keywordType as keyword +import org.python.antlr.ast.modType as mod +import org.python.antlr.ast.operatorType as operator +import org.python.antlr.ast.sliceType as slice +import org.python.antlr.ast.stmtType as stmt +import org.python.antlr.ast.unaryopType as unaryop +import org.python.antlr.PythonTree as AST Added: trunk/jython/src/org/python/antlr/IParserHost.java =================================================================== --- trunk/jython/src/org/python/antlr/IParserHost.java (rev 0) +++ trunk/jython/src/org/python/antlr/IParserHost.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -0,0 +1,23 @@ +package org.python.antlr; + +/** + * + * literal creation callbacks from the parser to the its host + * + **/ + +public interface IParserHost { + + public Object newLong(String s); + + public Object newLong(java.math.BigInteger i); + + public Object newFloat(double v); + + public Object newImaginary(double v); + + public Object newInteger(int i); + + public String decode_UnicodeEscape(String str, int start, int end, + String errors, boolean unicode); +} Added: trunk/jython/src/org/python/antlr/PythonGrammar.java =================================================================== --- trunk/jython/src/org/python/antlr/PythonGrammar.java (rev 0) +++ trunk/jython/src/org/python/antlr/PythonGrammar.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -0,0 +1,107 @@ +package org.python.antlr; + +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.Token; +import org.antlr.runtime.tree.CommonTree; +import org.antlr.runtime.tree.CommonTreeAdaptor; +import org.antlr.runtime.tree.CommonTreeNodeStream; +import org.antlr.runtime.tree.Tree; +import org.antlr.runtime.tree.TreeAdaptor; +import org.python.antlr.ast.modType; + +public class PythonGrammar { + public static class PyLexer extends PythonLexer { + public PyLexer(CharStream lexer) { + super(lexer); + } + + public Token nextToken() { + startPos = getCharPositionInLine(); + return super.nextToken(); + } + } + public static TreeAdaptor pyadaptor = new CommonTreeAdaptor() { + public Object create(Token token) { + /* + * if (token != null && token.getType() == PythonParser.Target) { + * System.out.println("Target found"); } + */ + return new PythonTree(token); + } + + public Object dupNode(Object t) { + if (t == null) { + return null; + } + return create(((PythonTree) t).token); + } + }; + + + private CharStream charStream; + private boolean partial; + + public PythonGrammar(CharStream cs) { + this(cs, false); + } + + public PythonGrammar(CharStream cs, boolean partial) { + this.partial = partial; + this.charStream = cs; + } + + public void printTree(CommonTree t, int indent) { + if (t != null) { + System.out.println("XXX: " + t.toString() + t.getType()); + StringBuffer sb = new StringBuffer(indent); + for (int i = 0; i < indent; i++) { + sb = sb.append(" "); + } + for (int i = 0; i < t.getChildCount(); i++) { + System.out.println(sb.toString() + t.getChild(i).toString() + ":" + t.getChild(i).getType()); + printTree((CommonTree) t.getChild(i), indent + 1); + } + } + } + + public modType file_input() { + modType tree = null; + // CharStream input = new ANTLRFileStream(args[0]); + PythonLexer lexer = new PyLexer(this.charStream); + CommonTokenStream tokens = new CommonTokenStream(lexer); + tokens.discardOffChannelTokens(true); + PythonTokenSource indentedSource = new PythonTokenSource(tokens); + tokens = new CommonTokenStream(indentedSource); + // System.out.println("tokens="+tokens.getTokens()); + PythonParser parser = new PythonParser(tokens); + parser.setTreeAdaptor(pyadaptor); + try { + //PythonParser.file_input_return r = parser.file_input(); + Object rx = parser.file_input(); + PythonParser.file_input_return r = (PythonParser.file_input_return)rx; + System.out.println("tree: " + ((Tree) r.tree).toStringTree()); + System.out.println("-----------------------------------"); + //printTree((CommonTree) r.tree, 0); + CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree); + nodes.setTokenStream(tokens); + PythonWalker walker = new PythonWalker(nodes); + tree = walker.module(); + } catch (RecognitionException e) { + // FIXME: + System.err.println("FIXME: don't eat exceptions:" + e); + } + return tree; + } + + public modType eval_input() { + // TODO Auto-generated method stub + return null; + } + + public modType single_input() { + return file_input(); + } + +} Modified: trunk/jython/src/org/python/antlr/PythonTreeWalker.java =================================================================== --- trunk/jython/src/org/python/antlr/PythonTreeWalker.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/antlr/PythonTreeWalker.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -46,7 +46,7 @@ } } if (args.length > 1) { - System.out.println(((Tree) r.tree).toStringTree()); + //System.out.println(((Tree) r.tree).toStringTree()); } if (!isParseOnly()) { CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree) r.tree); @@ -54,7 +54,7 @@ PythonWalker walker = new PythonWalker(nodes); result = walker.module(); if (args.length > 1) { - System.out.println(result.toStringTree()); + //System.out.println(result.toStringTree()); } } } catch (RecognitionException e) { Modified: trunk/jython/src/org/python/core/CompilerFlags.java =================================================================== --- trunk/jython/src/org/python/core/CompilerFlags.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/core/CompilerFlags.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -2,6 +2,13 @@ package org.python.core; public class CompilerFlags { + + public boolean nested_scopes = true; + public boolean division; + public boolean generator_allowed = true; + public boolean only_ast = false; + + public String encoding; public CompilerFlags(){} @@ -15,6 +22,9 @@ if ((co_flags & org.python.core.PyTableCode.CO_GENERATOR_ALLOWED) != 0) { this.generator_allowed = true; } + if ((co_flags & org.python.core.PyTableCode.PyCF_ONLY_AST) != 0) { + this.only_ast = true; + } } public String toString() { @@ -22,11 +32,4 @@ + generator_allowed + "]"; } - - - public boolean nested_scopes = true; - public boolean division; - public boolean generator_allowed = true; - - public String encoding; } Modified: trunk/jython/src/org/python/core/Py.java =================================================================== --- trunk/jython/src/org/python/core/Py.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/core/Py.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -1195,7 +1195,7 @@ } else throw Py.TypeError( "exec: argument 1 must be string, code or file object"); - code = Py.compile_flags(contents, "<string>", "exec", + code = (PyCode)Py.compile_flags(contents, "<string>", "exec", Py.getCompilerFlags()); } Py.runCode(code, locals, globals); @@ -1564,17 +1564,17 @@ // w/o compiler-flags - public static PyCode compile(modType node, String filename) { + public static PyObject compile(modType node, String filename) { return compile(node, getName(), filename); } - public static PyCode compile(modType node, String name, + public static PyObject compile(modType node, String name, String filename) { return compile(node, name, filename, true, false); } - public static PyCode compile(modType node, String name, + public static PyObject compile(modType node, String name, String filename, boolean linenumbers, boolean printResults) @@ -1583,7 +1583,7 @@ printResults, null); } - public static PyCode compile(InputStream istream, String filename, + public static PyObject compile(InputStream istream, String filename, String type) { return compile_flags(istream,filename,type,null); @@ -1591,7 +1591,7 @@ // with compiler-flags - public static PyCode compile_flags(modType node, String name, + public static PyObject compile_flags(modType node, String name, String filename, boolean linenumbers, boolean printResults,CompilerFlags cflags) @@ -1609,9 +1609,14 @@ } } - public static PyCode compile_flags(InputStream istream, String filename, + public static PyObject compile_flags(InputStream istream, String filename, String type,CompilerFlags cflags) { + if (cflags.only_ast) { + org.python.antlr.ast.modType node = antlr.parse(istream, type, filename, cflags); + return Py.java2py(node); + } + modType node = parser.parse(istream, type, filename, cflags); boolean printResults = false; if (type.equals("single")) @@ -1620,7 +1625,7 @@ cflags); } - public static PyCode compile_flags(String data, + public static PyObject compile_flags(String data, String filename, String type, CompilerFlags cflags) { Modified: trunk/jython/src/org/python/core/PyTableCode.java =================================================================== --- trunk/jython/src/org/python/core/PyTableCode.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/core/PyTableCode.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -23,18 +23,22 @@ int func_id; public String co_code = ""; // only used by inspect - final public static int CO_OPTIMIZED = 0x0001; - //final public static int CO_NEWLOCALS = 0x0002 - final public static int CO_VARARGS = 0x0004; - final public static int CO_VARKEYWORDS = 0x0008; - final public static int CO_GENERATOR = 0x0020; + final public static int CO_OPTIMIZED = 0x0001; + //final public static int CO_NEWLOCALS = 0x0002 + final public static int CO_VARARGS = 0x0004; + final public static int CO_VARKEYWORDS = 0x0008; + final public static int CO_GENERATOR = 0x0020; - final public static int CO_NESTED = 0x0010; + final public static int CO_NESTED = 0x0010; final public static int CO_GENERATOR_ALLOWED = 0x1000; - final public static int CO_FUTUREDIVISION = 0x2000; - final public static int CO_ALL_FEATURES = CO_NESTED|CO_GENERATOR_ALLOWED|CO_FUTUREDIVISION; + final public static int CO_FUTUREDIVISION = 0x2000; + //XXX: I'm not positive that this is the right place for this constant. + final public static int PyCF_ONLY_AST = 0x0400; + final public static int CO_ALL_FEATURES = PyCF_ONLY_AST|CO_NESTED|CO_GENERATOR_ALLOWED| + CO_FUTUREDIVISION; + public PyTableCode(int argcount, String varnames[], String filename, String name, int firstlineno, Modified: trunk/jython/src/org/python/core/__builtin__.java =================================================================== --- trunk/jython/src/org/python/core/__builtin__.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/core/__builtin__.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -457,11 +457,11 @@ throw Py.TypeError("number coercion failed"); } - public static PyCode compile(String data, String filename, String type) { + public static PyObject compile(String data, String filename, String type) { return Py.compile_flags(data, filename, type, Py.getCompilerFlags()); } - public static PyCode compile(String data, String filename, String type, int flags, boolean dont_inherit) { + public static PyObject compile(String data, String filename, String type, int flags, boolean dont_inherit) { if ((flags & ~PyTableCode.CO_ALL_FEATURES) != 0) { throw Py.ValueError("compile(): unrecognised flags"); } @@ -507,7 +507,7 @@ code = (PyCode) o; } else { if (o instanceof PyString) { - code = compile(o.toString(), "<string>", "eval"); + code = (PyCode)compile(o.toString(), "<string>", "eval"); } else { throw Py.TypeError("eval: argument 1 must be string or code object"); } @@ -540,7 +540,7 @@ PyCode code; try { - code = Py.compile_flags(file, name, "exec", cflags); + code = (PyCode)Py.compile_flags(file, name, "exec", cflags); } finally { try { file.close(); Added: trunk/jython/src/org/python/core/antlr.java =================================================================== --- trunk/jython/src/org/python/core/antlr.java (rev 0) +++ trunk/jython/src/org/python/core/antlr.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -0,0 +1,248 @@ +// Copyright (c) Corporation for National Research Initiatives +package org.python.core; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.UnsupportedEncodingException; + +import org.antlr.runtime.ANTLRReaderStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.RecognitionException; +import org.python.antlr.PythonGrammar; +import org.python.core.util.StringUtil; +import org.python.antlr.IParserHost; +import org.python.antlr.PythonTree; +import org.python.antlr.ast.modType; + +/** + * Facade for the classes in the org.python.antlr package. + */ + +public class antlr { + + private static IParserHost literalMkrForParser = new LiteralMakerForParser2(); + + private antlr() {} + + static String getLine(BufferedReader reader, int line) { + if (reader == null) + return ""; + try { + String text=null; + for(int i=0; i < line; i++) { + text = reader.readLine(); + } + return text; + } catch (IOException ioe) { + return null; + } + } + + // if reader != null, reset it + public static PyException fixParseError(BufferedReader reader, Throwable t, + String filename) + { + if (reader != null) { + try { + reader.reset(); + } catch (IOException e) { + reader = null; + } + } + + if (t instanceof RecognitionException) { + RecognitionException e = (RecognitionException)t; + //FJW Token tok = e.currentToken; + int col=0; + int line=0; + //FJW if (tok != null && tok.next != null) { + //FJW col = tok.next.beginColumn; + //FJW line = tok.next.beginLine; + //FJW } + String text=getLine(reader, line); + return new PySyntaxError(e.getMessage(), line, col, + text, filename); + } + /* FJW FJW FJW + if (t instanceof TokenMgrError) { + TokenMgrError e = (TokenMgrError)t; + boolean eofSeen = e.EOFSeen; + + int col = e.errorColumn; + int line = e.errorLine; + String text = getLine(reader, line); + if (eofSeen) + col -= 1; + return new PySyntaxError(e.getMessage(), line, col, + text, filename); + } + */ + else return Py.JavaError(t); + } + + public static PythonTree parse(String string, String kind) { + return parse(new ByteArrayInputStream(StringUtil.toBytes(string)), + kind, "<string>", null); + } + + public static modType parse(InputStream istream, String kind, + String filename, CompilerFlags cflags) + { + BufferedReader bufreader = prepBufreader(istream, cflags); + /* FJW FJW + PythonGrammar g = new PythonGrammar(new ReaderCharStream(bufreader), + literalMkrForParser); + */ + CharStream cs = null; + try { + cs = new ANTLRReaderStream(bufreader); + } catch (IOException io){ + //FIXME: + System.err.println("FIXME: Don't eat exceptions."); + } + PythonGrammar g = new PythonGrammar(cs);//FJW, literalMkrForParser); + modType node = null; + try { + node = doparse(kind, cflags, g); + } + catch (Throwable t) { + throw fixParseError(bufreader, t, filename); + } + return node; + } + + public static modType partialParse(String string, String kind, + String filename, CompilerFlags cflags,boolean stdprompt) + { + modType node = null; + BufferedReader bufreader = prepBufreader(new ByteArrayInputStream(StringUtil.toBytes(string)), + cflags); + + CharStream cs = null; + try { + cs = new ANTLRReaderStream(bufreader); + } catch (IOException io){ + //FIXME: + System.err.println("FIXME: Don't eat exceptions."); + } + PythonGrammar g = new PythonGrammar(cs, true); + //FJW g.token_source.partial = true; + //FJW g.token_source.stdprompt = stdprompt; + + try { + node = doparse(kind, cflags, g); + } + catch (Throwable t) { + /* + CPython codeop exploits that with CPython parser adding newlines + to a partial valid sentence move the reported error position, + this is not true for our parser, so we need a different approach: + we check whether all sentence tokens have been consumed or + the remaining ones fullfill lookahead expectations. See: + PythonGrammar.partial_valid_sentence (def in python.jjt) + */ + + //FJW if (g.partial_valid_sentence(t)) { + //FJW return null; + //FJW } + throw fixParseError(bufreader, t, filename); + } + return node; + } + + private static modType doparse(String kind, CompilerFlags cflags, + PythonGrammar g) throws /*Parse*/Exception //FJW + { + modType node = null; + + //FJW if (cflags != null) + //FJW g.token_source.generator_allowed = cflags.generator_allowed; + + if (kind.equals("eval")) { + node = g.eval_input(); + } + else if (kind.equals("exec")) { + node = g.file_input(); + } + else if (kind.equals("single")) { + node = g.single_input(); + } + else { + throw Py.ValueError("parse kind must be eval, exec, " + + "or single"); + } + return node; + } + + private static BufferedReader prepBufreader(InputStream istream, + CompilerFlags cflags) { + int nbytes; + try { + nbytes = istream.available(); + } + catch (IOException ioe1) { + nbytes = 10000; + } + if (nbytes <= 0) + nbytes = 10000; + if (nbytes > 100000) + nbytes = 100000; + Reader reader; + if(cflags != null && cflags.encoding != null) { + try { + reader = new InputStreamReader(istream, cflags.encoding); + } catch(UnsupportedEncodingException exc) { + throw Py.SystemError("python.console.encoding, " + cflags.encoding + + ", isn't supported by this JVM so we can't parse this data."); + } + } else { + try { + // Use ISO-8859-1 to get bytes off the input stream since it leaves their values alone. + reader = new InputStreamReader(istream, "ISO-8859-1"); + } catch(UnsupportedEncodingException e) { + // This JVM is whacked, it doesn't even have iso-8859-1 + throw Py.SystemError("Java couldn't find the ISO-8859-1 encoding"); + } + } + + BufferedReader bufreader = new BufferedReader(reader); + + try { + bufreader.mark(nbytes); + } catch (IOException exc) { } + return bufreader; + } +} + +class LiteralMakerForParser2 implements IParserHost { + + public Object newLong(String s) { + return Py.newLong(s); + } + + public Object newLong(java.math.BigInteger i) { + return Py.newLong(i); + } + + public Object newFloat(double v) { + return Py.newFloat(v); + } + + public Object newImaginary(double v) { + return Py.newImaginary(v); + } + + public Object newInteger(int i) { + return Py.newInteger(i); + } + + public String decode_UnicodeEscape( + String str, int start, int end, String errors, boolean unicode) { + return PyString.decode_UnicodeEscape(str, start, end, errors, unicode); + } + +} Modified: trunk/jython/src/org/python/util/PythonInterpreter.java =================================================================== --- trunk/jython/src/org/python/util/PythonInterpreter.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/util/PythonInterpreter.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -149,7 +149,7 @@ public void execfile(java.io.InputStream s, String name) { setState(); - Py.runCode(Py.compile_flags(s, name, "exec",cflags), locals, locals); + Py.runCode((PyCode)Py.compile_flags(s, name, "exec",cflags), locals, locals); } // Getting and setting the locals dictionary Modified: trunk/jython/src/org/python/util/jython.java =================================================================== --- trunk/jython/src/org/python/util/jython.java 2008-04-10 02:45:22 UTC (rev 4331) +++ trunk/jython/src/org/python/util/jython.java 2008-04-11 00:09:27 UTC (rev 4332) @@ -82,7 +82,7 @@ InputStream file = zip.getInputStream(runit); PyCode code; try { - code = Py.compile(file, "__run__", "exec"); + code = (PyCode)Py.compile(file, "__run__", "exec"); } finally { file.close(); } This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |