Thread: [Jython-checkins] SF.net SVN: jython: [4442] trunk/jython/src/org/python (Page 2)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 454-5900

Revision: 4442
          http://jython.svn.sourceforge.net/jython/?rev=4442&view=rev
Author:   fwierzbicki
Date:     2008-05-24 07:56:35 -0700 (Sat, 24 May 2008)

Log Message:
-----------
Broke out antlr parsing of Expressions (aka: evals).  Start of a
fix for evals with leading spaces.  Extended PythonTreeWalker so 
eval and interactive parsing is supported.

Modified Paths:
--------------
    trunk/jython/src/org/python/antlr/PythonGrammar.java
    trunk/jython/src/org/python/antlr/PythonTreeWalker.java
    trunk/jython/src/org/python/core/antlr.java

Added Paths:
-----------
    trunk/jython/src/org/python/antlr/ExpressionParser.java
    trunk/jython/src/org/python/antlr/LeadingSpaceSkippingStream.java

Added: trunk/jython/src/org/python/antlr/ExpressionParser.java
===================================================================

--- trunk/jython/src/org/python/antlr/ExpressionParser.java	                        (rev 0)
+++ trunk/jython/src/org/python/antlr/ExpressionParser.java	2008-05-24 14:56:35 UTC (rev 4442)
@@ -0,0 +1,73 @@
+package org.python.antlr;
+
+import org.antlr.runtime.CharStream;
+import org.antlr.runtime.CommonToken;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.Token;
+import org.antlr.runtime.tree.CommonTree;
+import org.antlr.runtime.tree.CommonTreeAdaptor;
+import org.antlr.runtime.tree.CommonTreeNodeStream;
+import org.antlr.runtime.tree.Tree;
+import org.antlr.runtime.tree.TreeAdaptor;
+import org.python.antlr.ast.modType;
+import org.python.antlr.ast.Module;
+import org.python.antlr.ast.stmtType;
+
+public class ExpressionParser {
+
+    private CharStream charStream;
+
+    //Extract superclass from this and the other XParsers.
+    public static class PyLexer extends PythonLexer {
+        public PyLexer(CharStream lexer) {
+            super(lexer);
+        }
+
+        public Token nextToken() {
+            startPos = getCharPositionInLine();
+            return super.nextToken();
+        }
+    }
+
+    public static TreeAdaptor pyadaptor = new CommonTreeAdaptor() {
+        public Object create(Token token) {
+            return new PythonTree(token);
+        }
+
+        public Object dupNode(Object t) {
+            if (t == null) {
+                return null;
+            }
+            return create(((PythonTree) t).token);
+        }
+    };
+
+    public ExpressionParser(CharStream cs) {
+        this.charStream = cs;
+    }
+
+    public modType parse() {
+        modType tree = null;
+        PythonLexer lexer = new PyLexer(this.charStream);
+        CommonTokenStream tokens = new CommonTokenStream(lexer);
+        tokens.discardOffChannelTokens(true);
+        PythonTokenSource indentedSource = new PythonTokenSource(tokens);
+        tokens = new CommonTokenStream(indentedSource);
+        PythonParser parser = new PythonParser(tokens);
+        parser.setTreeAdaptor(pyadaptor);
+
+        try {
+            Object rx = parser.eval_input();
+            PythonParser.eval_input_return r = (PythonParser.eval_input_return)rx;
+            CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);
+            nodes.setTokenStream(tokens);
+            PythonWalker walker = new PythonWalker(nodes);
+            tree = walker.expression();
+        } catch (RecognitionException e) {
+            // FIXME:
+            System.err.println("FIXME: don't eat exceptions:" + e);
+        }
+        return tree;
+    } 
+}

Added: trunk/jython/src/org/python/antlr/LeadingSpaceSkippingStream.java
===================================================================
--- trunk/jython/src/org/python/antlr/LeadingSpaceSkippingStream.java	                        (rev 0)
+++ trunk/jython/src/org/python/antlr/LeadingSpaceSkippingStream.java	2008-05-24 14:56:35 UTC (rev 4442)
@@ -0,0 +1,26 @@
+package org.python.antlr;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+public class LeadingSpaceSkippingStream extends InputStream {
+
+    private InputStream inputStream;
+    private boolean maybeLeadingSpaces = true;
+
+    public LeadingSpaceSkippingStream(InputStream is) {
+        inputStream = is;
+    }
+
+    public int read() throws IOException {
+        int i = inputStream.read();
+        while (maybeLeadingSpaces) {
+            if (i != ' ') {
+                maybeLeadingSpaces = false;
+            } else {
+                i = inputStream.read();
+            }
+        }
+        return i;
+    }
+}

Modified: trunk/jython/src/org/python/antlr/PythonGrammar.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonGrammar.java	2008-05-24 14:34:05 UTC (rev 4441)
+++ trunk/jython/src/org/python/antlr/PythonGrammar.java	2008-05-24 14:56:35 UTC (rev 4442)
@@ -86,7 +86,7 @@
             tree = walker.module();
             if (tree == null) {
                 //XXX: seems like I should be able to get antlr to give me an empty Module instead
-                //     of null so I wouldn't need to build an empty Moduel by hand here...
+                //     of null so I wouldn't need to build an empty Module by hand here...
                 return new Module(new PythonTree(new CommonToken(PyLexer.Module)), new stmtType[0]);
             }
         } catch (RecognitionException e) {
@@ -97,31 +97,6 @@
     }
 
     //XXX: factor out common code.
-    public modType eval_input() {
-        modType tree = null;
-        PythonLexer lexer = new PyLexer(this.charStream);
-        CommonTokenStream tokens = new CommonTokenStream(lexer);
-        tokens.discardOffChannelTokens(true);
-        PythonTokenSource indentedSource = new PythonTokenSource(tokens);
-        tokens = new CommonTokenStream(indentedSource);
-        PythonParser parser = new PythonParser(tokens);
-        parser.setTreeAdaptor(pyadaptor);
-
-        try {
-            Object rx = parser.eval_input();
-            PythonParser.eval_input_return r = (PythonParser.eval_input_return)rx;
-            CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree)r.tree);
-            nodes.setTokenStream(tokens);
-            PythonWalker walker = new PythonWalker(nodes);
-            tree = walker.expression();
-        } catch (RecognitionException e) {
-            // FIXME:
-            System.err.println("FIXME: don't eat exceptions:" + e);
-        }
-        return tree;
-    }
-
-    //XXX: factor out common code.
     public modType single_input() {
         modType tree = null;
         PythonLexer lexer = new PyLexer(this.charStream);

Modified: trunk/jython/src/org/python/antlr/PythonTreeWalker.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTreeWalker.java	2008-05-24 14:34:05 UTC (rev 4441)
+++ trunk/jython/src/org/python/antlr/PythonTreeWalker.java	2008-05-24 14:56:35 UTC (rev 4442)
@@ -15,12 +15,16 @@
  */
 public class PythonTreeWalker {
 
+    public enum Block { MODULE, INTERACTIVE, EXPRESSION };
+
 	private boolean _parseOnly;
 	private boolean _tolerant;
+    private Block _block;
 
 	public PythonTreeWalker() {
 		setParseOnly(false);
 		setTolerant(true);
+        setBlock(Block.MODULE);
 	}
 
 	public PythonTree parse(String[] args) throws Exception {
@@ -31,11 +35,22 @@
 		tokens.discardOffChannelTokens(true);
 		PythonTokenSource indentedSource = new PythonTokenSource(tokens);
 		tokens = new CommonTokenStream(indentedSource);
-		// System.out.println("tokens="+tokens.getTokens());
 		PythonParser parser = new PythonParser(tokens);
 		parser.setTreeAdaptor(PythonGrammar.pyadaptor);
 		try {
-			PythonParser.file_input_return r = parser.file_input();
+            Tree r = null;
+            switch (_block) {
+            case MODULE :
+			    r = (Tree)parser.file_input().tree;
+                break;
+            case INTERACTIVE :
+			    r = (Tree)parser.single_input().tree;
+                break;
+            case EXPRESSION :
+			    r = (Tree)parser.eval_input().tree;
+                break;
+            }
+			//Tree r = (Tree)parser.file_input().tree;
 			if (parser.hasErrors()) {
 				// handle errors swallowed by antlr recovery
 				String errors = parser.getErrors().toString();
@@ -46,13 +61,24 @@
 				}
 			}
 			if (args.length > 1) {
-				System.out.println(((Tree) r.tree).toStringTree());
+				System.out.println((r).toStringTree());
 			}
 			if (!isParseOnly()) {
-				CommonTreeNodeStream nodes = new CommonTreeNodeStream((Tree) r.tree);
+				CommonTreeNodeStream nodes = new CommonTreeNodeStream(r);
 				nodes.setTokenStream(tokens);
 				PythonWalker walker = new PythonWalker(nodes);
-				result = walker.module();
+                switch (_block) {
+                case MODULE :
+				    result = walker.module();
+                    break;
+                case INTERACTIVE :
+                    result = walker.interactive();
+                    break;
+                case EXPRESSION :
+                    result = walker.expression();
+                    break;
+                }
+
 				if (args.length > 1) {
 					System.out.println(result.toStringTree());
 				}
@@ -95,4 +121,12 @@
 		return _tolerant;
 	}
 
+	public void setBlock(Block block) {
+		_block = block;
+	}
+
+	public Block getBlock() {
+		return _block;
+	}
+
 }

Modified: trunk/jython/src/org/python/core/antlr.java
===================================================================
--- trunk/jython/src/org/python/core/antlr.java	2008-05-24 14:34:05 UTC (rev 4441)
+++ trunk/jython/src/org/python/core/antlr.java	2008-05-24 14:56:35 UTC (rev 4442)
@@ -12,6 +12,8 @@
 import org.antlr.runtime.ANTLRReaderStream;
 import org.antlr.runtime.CharStream;
 import org.antlr.runtime.RecognitionException;
+import org.python.antlr.ExpressionParser;
+import org.python.antlr.LeadingSpaceSkippingStream;
 import org.python.antlr.PythonGrammar;
 import org.python.core.util.StringUtil;
 import org.python.antlr.IParserHost;
@@ -92,22 +94,23 @@
     public static modType parse(InputStream istream, String kind,
                                  String filename, CompilerFlags cflags) 
     {
-        BufferedReader bufreader = prepBufreader(istream, cflags);
-        /* FJW FJW
-        PythonGrammar g = new PythonGrammar(new ReaderCharStream(bufreader),
-                                            literalMkrForParser);
-        */
         CharStream cs = null;
-        try {
-            cs = new ANTLRReaderStream(bufreader);
-        } catch (IOException io){
-            //FIXME:
-            System.err.println("FIXME: Don't eat exceptions.");
-        }
-        PythonGrammar g = new PythonGrammar(cs);//FJW, literalMkrForParser);
+        //FIXME: definite NPE potential here -- do we even need prepBufreader
+        //       now?
+        BufferedReader bufreader = null;
         modType node = null;
         try {
-            node = doparse(kind, cflags, g);
+            if (kind.equals("eval")) {
+                bufreader = prepBufreader(new LeadingSpaceSkippingStream(istream), cflags);
+                cs = new ANTLRReaderStream(bufreader);
+                ExpressionParser e = new ExpressionParser(cs);
+                node = e.parse();
+            } else {
+                bufreader = prepBufreader(istream, cflags);
+                cs = new ANTLRReaderStream(bufreader);
+                PythonGrammar g = new PythonGrammar(cs);//FJW, literalMkrForParser);
+                node = doparse(kind, cflags, g);
+            }
         }
         catch (Throwable t) {
             throw fixParseError(bufreader, t, filename);
@@ -162,10 +165,7 @@
         //FJW if (cflags != null)
         //FJW    g.token_source.generator_allowed = cflags.generator_allowed;
         
-        if (kind.equals("eval")) {
-            node = g.eval_input();
-        }
-        else if (kind.equals("exec")) {
+        if (kind.equals("exec")) {
             node = g.file_input();
         }
         else if (kind.equals("single")) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.




Thread: [Jython-checkins] SF.net SVN: jython: [4442] trunk/jython/src/org/python (Page 2)

jython-checkins