[Jython-checkins] SF.net SVN: jython:[5074] branches/asm

[<fwi...@us...>]

Revision: 5074
          http://jython.svn.sourceforge.net/jython/?rev=5074&view=rev
Author:   fwierzbicki
Date:     2008-08-04 01:15:48 +0000 (Mon, 04 Aug 2008)

Log Message:
-----------
Fixes to handle parsing with files that have no newline at the end.

Modified Paths:
--------------
    branches/asm/grammar/Python.g
    branches/asm/src/org/python/antlr/PythonTokenSource.java

Modified: branches/asm/grammar/Python.g
===================================================================
--- branches/asm/grammar/Python.g	2008-08-03 22:54:14 UTC (rev 5073)
+++ branches/asm/grammar/Python.g	2008-08-04 01:15:48 UTC (rev 5074)
@@ -1383,10 +1383,14 @@
                for (int i=0; i<spaces; i++) {
                    indentation[i] = ' ';
                }
-               CommonToken c = new CommonToken(LEADING_WS,new String(indentation));
-               c.setLine(input.getLine());
-               c.setCharPositionInLine(input.getCharPositionInLine());
-               emit(c);
+               if (input.LA(1) != -1) {
+                   CommonToken c = new CommonToken(LEADING_WS,new String(indentation));
+                   c.setLine(input.getLine());
+                   c.setCharPositionInLine(input.getCharPositionInLine());
+                   emit(c);
+               } else {
+                   emit(new CommonToken(LEADING_WS,""));
+               }
             }
             // kill trailing newline if present and then ignore
             ( ('\r')? '\n' {if (state.token!=null) state.token.setChannel(HIDDEN); else $channel=HIDDEN;})*

Modified: branches/asm/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-03 22:54:14 UTC (rev 5073)
+++ branches/asm/src/org/python/antlr/PythonTokenSource.java	2008-08-04 01:15:48 UTC (rev 5074)
@@ -179,8 +179,8 @@
             newline = new CommonToken(PythonLexer.NEWLINE, "\n");
             newline.setLine(t.getLine());
             newline.setCharPositionInLine(t.getCharPositionInLine());
-            //XXX: should be moved to after DEDENTS are generated, but doesn't work yet.
-            tokens.addElement(newline);
+            //XXX: this is where lsoto had this...
+            //tokens.addElement(newline);
         }
 
         // compute cpos as the char pos of next non-WS token in line
@@ -223,13 +223,10 @@
             }
             sp = prevIndex; // pop those off indent level
         }
-        /*
-        //XXX: I think this is really where this needs to be, but causes more problems than it
-        //     solves at the moment.
+        //XXX: make sure lsoto's stuff isn't broken by this...
         if (t.getType() == PythonLexer.EOF) {
             tokens.addElement(newline);
         }
-        */
 
         if (t.getType() != PythonLexer.LEADING_WS) { // discard WS
             tokens.addElement(t);


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.