[Jython-checkins] SF.net SVN: jython:[5288] trunk/jython

[<fwi...@us...>]

Revision: 5288
          http://jython.svn.sourceforge.net/jython/?rev=5288&view=rev
Author:   fwierzbicki
Date:     2008-09-04 16:57:30 +0000 (Thu, 04 Sep 2008)

Log Message:
-----------
Fix for some offset problems -- the biggest one being dedents now take on the
offset values of the previous token.

Modified Paths:
--------------
    trunk/jython/grammar/Python.g
    trunk/jython/src/org/python/antlr/PythonTokenSource.java
    trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java

Modified: trunk/jython/grammar/Python.g
===================================================================
--- trunk/jython/grammar/Python.g	2008-09-03 15:24:20 UTC (rev 5287)
+++ trunk/jython/grammar/Python.g	2008-09-04 16:57:30 UTC (rev 5288)
@@ -284,6 +284,12 @@
     List stypes = new ArrayList();
 }
 @after {
+    if (!stypes.isEmpty()) {
+        //The EOF token messes up the end offsets, so set them manually.
+        PythonTree stop = (PythonTree)stypes.get(stypes.size() -1);
+        mtype.setCharStopIndex(stop.getCharStopIndex());
+        mtype.setTokenStopIndex(stop.getTokenStopIndex());
+    }
     $file_input.tree = mtype;
 }
     : (NEWLINE

Modified: trunk/jython/src/org/python/antlr/PythonTokenSource.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-03 15:24:20 UTC (rev 5287)
+++ trunk/jython/src/org/python/antlr/PythonTokenSource.java	2008-09-04 16:57:30 UTC (rev 5288)
@@ -157,14 +157,18 @@
         stream.consume();
 
         if (t.getType() == Token.EOF) {
+            Token prev = stream.LT(-1);
             if (!inSingle) {
-                Token prev = stream.LT(-1);
                 if (prev == null || prev.getType() != PythonLexer.NEWLINE) {
                     generateNewline(t);
                 }
             }
-
-            handleDedents(-1, (CommonToken)t);
+            if (prev != null) {
+                ((CommonToken)t).setStopIndex(((CommonToken)prev).getStopIndex());
+                handleDedents(-1, (CommonToken)prev);
+            } else {
+                handleDedents(-1, (CommonToken)t);
+            }
             enqueue(t);
         } else if (t.getType() == PythonLexer.NEWLINE) {
             // save NEWLINE in the queue
@@ -203,7 +207,12 @@
                 handleIndents(cpos, (CommonToken)t);
             }
             else if (cpos < lastIndent) { // they dedented
-                handleDedents(cpos, (CommonToken)t);
+                Token prev = stream.LT(-1);
+                if (prev != null) {
+                    handleDedents(cpos, (CommonToken)prev);
+                } else {
+                    handleDedents(cpos, (CommonToken)t);
+                }
             }
 
             if (t.getType() == Token.EOF && inSingle) {

Modified: trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java
===================================================================
--- trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java	2008-09-03 15:24:20 UTC (rev 5287)
+++ trunk/jython/src/org/python/antlr/PythonTreeAdaptor.java	2008-09-04 16:57:30 UTC (rev 5288)
@@ -12,6 +12,8 @@
         if (t==null) {
             return;
         }
+		//System.out.println("setting boundries on '"+t+"' with start: '" + startToken + "' and '" +
+        //        stopToken + "'");
         int start = 0;
         int stop = 0;
         int startChar = 0;


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.