[Jython-checkins] SF.net SVN: jython:[5291] trunk/jython

[<fwi...@us...>]

Revision: 5291
          http://jython.svn.sourceforge.net/jython/?rev=5291&view=rev
Author:   fwierzbicki
Date:     2008-09-05 16:30:36 +0000 (Fri, 05 Sep 2008)

Log Message:
-----------
Broke test_traceback.py in my last commit.  This fixes it again.

Modified Paths:
--------------
    trunk/jython/grammar/Python.g
    trunk/jython/src/org/python/antlr/PythonTokenSource.java

Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g	2008-09-05 16:21:55 UTC (rev 5290)
+++ trunk/jython/grammar/Python.g	2008-09-05 16:30:36 UTC (rev 5291)
@@ -286,10 +286,14 @@
 @after {
     if (!stypes.isEmpty()) {
         //The EOF token messes up the end offsets, so set them manually.
+        //XXX: this may no longer be true now that PythonTokenSource is
+        //     adjusting EOF offsets -- but needs testing before I remove
+        //     this.
         PythonTree stop = (PythonTree)stypes.get(stypes.size() -1);
         mtype.setCharStopIndex(stop.getCharStopIndex());
         mtype.setTokenStopIndex(stop.getTokenStopIndex());
     }
+
     $file_input.tree = mtype;
 }
     : (NEWLINE
@@ -1778,6 +1782,8 @@
                        CommonToken c = new CommonToken(LEADING_WS,new String(indentation));
                        c.setLine(input.getLine());
                        c.setCharPositionInLine(input.getCharPositionInLine());
+                       c.setStartIndex(input.index() - 1);
+                       c.setStopIndex(input.index() - 1);
                        emit(c);
                        // kill trailing newline if present and then ignore
                        if (newlines != 0) {
@@ -1793,7 +1799,12 @@
                        for (int i=0; i<newlines; i++) {
                            nls[i] = '\n';
                        }
-                       emit(new CommonToken(NEWLINE,new String(nls)));
+                       CommonToken c = new CommonToken(NEWLINE,new String(nls));
+                       c.setLine(input.getLine());
+                       c.setCharPositionInLine(input.getCharPositionInLine());
+                       c.setStartIndex(input.index() - 1);
+                       c.setStopIndex(input.index() - 1);
+                       emit(c);
                    }
                 }
         )

Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-05 16:21:55 UTC (rev 5290)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-05 16:30:36 UTC (rev 5291)
@@ -152,23 +152,25 @@
         tokens.addElement(newline);
     }
 
+    private void handleEOF(CommonToken eof, CommonToken prev) {
+        if (prev != null) {
+            eof.setStopIndex(prev.getStopIndex());
+        }
+    }
+
     protected void insertImaginaryIndentDedentTokens() {
         Token t = stream.LT(1);
         stream.consume();
 
         if (t.getType() == Token.EOF) {
             Token prev = stream.LT(-1);
+            handleEOF((CommonToken)t, (CommonToken)prev);
             if (!inSingle) {
                 if (prev == null || prev.getType() != PythonLexer.NEWLINE) {
                     generateNewline(t);
                 }
             }
-            if (prev != null) {
-                ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex());
-                handleDedents(-1, (CommonToken)prev);
-            } else {
-                handleDedents(-1, (CommonToken)t);
-            }
+            handleDedents(-1, (CommonToken)t);
             enqueue(t);
         } else if (t.getType() == PythonLexer.NEWLINE) {
             // save NEWLINE in the queue
@@ -186,6 +188,7 @@
             // compute cpos as the char pos of next non-WS token in line
             int cpos = t.getCharPositionInLine(); // column dictates indent/dedent
             if (t.getType() == Token.EOF) {
+                handleEOF((CommonToken)t, (CommonToken)newline);
                 cpos = -1; // pretend EOF always happens at left edge
             }
             else if (t.getType() == PythonLexer.LEADING_WS) {
@@ -207,8 +210,7 @@
                 handleIndents(cpos, (CommonToken)t);
             }
             else if (cpos < lastIndent) { // they dedented
-                Token prev = stream.LT(-1);
-                handleDedents(cpos, (CommonToken)newline);
+                handleDedents(cpos, (CommonToken)t);
             }
 
             if (t.getType() == Token.EOF && inSingle) {


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.