[Jython-checkins] SF.net SVN: jython:[6593] trunk/jython

[<fwi...@us...>]

Revision: 6593
          http://jython.svn.sourceforge.net/jython/?rev=6593&view=rev
Author:   fwierzbicki
Date:     2009-07-27 19:30:01 +0000 (Mon, 27 Jul 2009)

Log Message:
-----------
Get our Lexer to properly fail for trailing whitespace in single mode.

Modified Paths:
--------------
    trunk/jython/Lib/test/test_codeop.py
    trunk/jython/grammar/Python.g
    trunk/jython/src/org/python/antlr/BaseParser.java

Modified: trunk/jython/Lib/test/test_codeop.py
===================================================================
--- trunk/jython/Lib/test/test_codeop.py	2009-07-27 17:22:32 UTC (rev 6592)
+++ trunk/jython/Lib/test/test_codeop.py	2009-07-27 19:30:01 UTC (rev 6593)
@@ -133,8 +133,8 @@
         ai("def x():\n\n")
 
         ai("def x():\n  pass")
-        #ai("def x():\n  pass\n ")
-        #ai("def x():\n  pass\n  ")
+        ai("def x():\n  pass\n ")
+        ai("def x():\n  pass\n  ")
         ai("\n\ndef x():\n  pass")
 
         ai("a = 9+ \\")

Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g	2009-07-27 17:22:32 UTC (rev 6592)
+++ trunk/jython/grammar/Python.g	2009-07-27 19:30:01 UTC (rev 6593)
@@ -209,6 +209,7 @@
 //For use in partial parsing.
 public boolean eofWhileNested = false;
 public boolean partial = false;
+public boolean single = false;
 
 int implicitLineJoiningLevel = 0;
 int startPos=-1;
@@ -1967,6 +1968,8 @@
                                $channel=HIDDEN;
                            }
                        }
+                   } else if (this.single && newlines == 1) {
+                       throw new ParseException("Trailing space in single mode.");
                    } else {
                        // make a string of n newlines
                        char[] nls = new char[newlines];

Modified: trunk/jython/src/org/python/antlr/BaseParser.java
===================================================================
--- trunk/jython/src/org/python/antlr/BaseParser.java	2009-07-27 17:22:32 UTC (rev 6592)
+++ trunk/jython/src/org/python/antlr/BaseParser.java	2009-07-27 19:30:01 UTC (rev 6593)
@@ -32,10 +32,16 @@
     }
 
     public static class PyLexer extends PythonLexer {
-        public PyLexer(CharStream lexer) {
+        public PyLexer(CharStream lexer, boolean single) {
             super(lexer);
+            this.single = single;
         }
 
+        public PyLexer(CharStream lexer) {
+            this(lexer, false);
+        }
+
+
         public Token nextToken() {
             startPos = getCharPositionInLine();
             return super.nextToken();
@@ -53,12 +59,8 @@
         }
     }
      
-    private CharStream charStream(boolean single) {
-        return charStream;
-    }
-
     private PythonParser setupParser(boolean single) {
-        PythonLexer lexer = new PyLexer(this.charStream(single));
+        PythonLexer lexer = new PyLexer(charStream, single);
         lexer.setErrorHandler(errorHandler);
         CommonTokenStream tokens = new CommonTokenStream(lexer);
         PythonTokenSource indentedSource = new PythonTokenSource(tokens, filename, single);


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.